Release v1.14.0

Fix the root-cause of #759 by introducing synchronisation points for
individual user fetches.

.golangci.yaml

@@ -132,6 +132,7 @@ linters-settings:
     # ifElseChain regexpMust singleCaseSwitch sloppyLen switchTrue typeSwitchVar underef
     # unlambda unslice rangeValCopy defaultCaseOrder];
     # all checks list: https://github.com/go-critic/checkers
+    # disabled for now - hugeParam
     enabled-checks:
       - appendAssign
       - assignOp
@@ -147,7 +148,6 @@ linters-settings:
       - dupSubExpr
       - elseif
       - emptyFallthrough
-      - hugeParam
       - ifElseChain
       - importShadow
       - indexAlloc
@@ -158,7 +158,6 @@ linters-settings:
       - regexpMust
       - singleCaseSwitch
       - sloppyLen
-      - sloppyReassign
       - switchTrue
       - typeSwitchVar
       - typeUnparen

.goreleaser.yml

@@ -0,0 +1,34 @@
+release:
+  prerelease: auto
+  name_template: "{{.ProjectName}} v{{.Version}}"
+
+builds:
+- env:
+    - CGO_ENABLED=0
+  goos:
+    - freebsd
+    - windows
+    - darwin
+    - linux
+    - dragonfly
+    - netbsd
+    - openbsd
+  goarch:
+    - amd64
+    - arm
+    - arm64
+    - 386
+  ldflags:
+    - -s -w -X main.githash={{.ShortCommit}}
+
+archive:
+  name_template: "{{ .Binary }}-{{ .Version }}-{{ .Os }}-{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}"
+  format: binary
+  files:
+    - none*
+  replacements:
+    386: 32bit
+    amd64: 64bit
+
+checksum:
+  name_template: 'checksums.txt'

.travis.yml

@@ -12,7 +12,8 @@ git:
 env:
   global:
     - GOOS=linux GOARCH=amd64
-    - GOLANGCI_VERSION="v1.12.3"
+    - GO111MODULE=on
+    - GOLANGCI_VERSION="v1.14.0"
 
 matrix:
   # It's ok if our code fails on unstable development versions of Go.
@@ -36,10 +37,17 @@ before_script:
   - ./cc-test-reporter before-build
 
 script:
+  # Ensure that the module files are being kept correctly and that vendored dependencies are up-to-date.
+  - go mod tidy
+  - go mod vendor
+  - git diff --exit-code --quiet || (echo "Please run 'go mod tidy' to clean up the 'go.mod' and 'go.sum' files."; false)
+
   # Run the linter.
   - golangci-lint run
+
   # Run all the tests with the race detector and generate coverage.
   - go test -v -race -coverprofile c.out ./...
+
   # Run the build script to generate the necessary binaries and images.
   - /bin/bash ci/bintray.sh
 
@@ -47,11 +55,21 @@ after_script:
   # Upload test coverage to CodeClimate.
   - ./cc-test-reporter after-build --exit-code ${TRAVIS_TEST_RESULT}
 
+branches:
+  only:
+    - master
+
 deploy:
+  on:
+     all_branches: true
   provider: bintray
+  on:
+    all_branches: true
   edge:
     branch: v1.8.47
   file: ci/deploy.json
   user: 42wim
+  on:
+     all_branches: true
   key:
-     secure: "CeXXe6JOmt7HYR81MdWLua0ltQHhDdkIeRGBFbgd7hkb1wi8eF9DgpAcQrTso8NIlHNZmSAP46uhFgsRvkuezzX0ygalZ7DCJyAyn3sAMEh+UQSHV1WGThRehTtidqRGjetzsIGSwdrJOWil+XTfbO1Z8DGzfakhSuAZka8CM4BAoe3YeP9rYK8h+84x0GHfczvsLtXZ3mWLvQuwe4pK6+ItBCUg0ae7O7ZUpWHy0xQQkkWztY/6RAzXfaG7DuGjIw+20fhx3WOXRNpHCtZ6Bc3qERCpk0s1HhlQWlrN9wDaFTBWYwlvSnNgvxxMbNXJ6RrRJ0l0bA7FUswYwyroxhzrGLdzWDg8dHaQkypocngdalfhpsnoO9j3ApJhomUFJ3UoEq5nOGRUrKn8MPi+dP0zE4kNQ3e4VNa1ufNrvfpWolMg3xh8OXuhQdD5wIM5zFAbRJLqWSCVAjPq4DDPecmvXBOlIial7oa312lN5qnBnUjvAcxszZ+FUyDHT1Grxzna4tMwxY9obPzZUzm7359AOCCwIQFVB8GLqD2nwIstcXS0zGRz+fhviPipHuBa02q5bGUZwmkvrSNab0s8Jo7pCrel2Rz3nWPKaiCfq2WjbW1CLheSMkOQrjsdUd1hhbqNWFPUjJPInTc77NAKCfm5runv5uyowRLh4NNd0sI="
+     secure: "CeXXe6JOmt7HYR81MdWLua0ltQHhDdkIeRGBFbgd7hkb1wi8eF9DgpAcQrTso8NIlHNZmSAP46uhFgsRvkuezzX0ygalZ7DCJyAyn3sAMEh+UQSHV1WGThRehTtidqRGjetzsIGSwdrJOWil+XTfbO1Z8DGzfakhSuAZka8CM4BAoe3YeP9rYK8h+84x0GHfczvsLtXZ3mWLvQuwe4pK6+ItBCUg0ae7O7ZUpWHy0xQQkkWztY/6RAzXfaG7DuGjIw+20fhx3WOXRNpHCtZ6Bc3qERCpk0s1HhlQWlrN9wDaFTBWYwlvSnNgvxxMbNXJ6RrRJ0l0bA7FUswYwyroxhzrGLdzWDg8dHaQkypocngdalfhpsnoO9j3ApJhomUFJ3UoEq5nOGRUrKn8MPi+dP0zE4kNQ3e4VNa1ufNrvfpWolMg3xh8OXuhQdD5wIM5zFAbRJLqWSCVAjPq4DDPecmvXBOlIial7oa312lN5qnBnUjvAcxszZ+FUyDHT1Grxzna4tMwxY9obPzZUzm7359AOCCwIQFVB8GLqD2nwIstcXS0zGRz+fhviPipHuBa02q5bGUZwmkvrSNab0s8Jo7pCrel2Rz3nWPKaiCfq2WjbW1CLheSMkOQrjsdUd1hhbqNWFPUjJPInTc77NAKCfm5runv5uyowRLh4NNd0sI="

README.md

@@ -5,7 +5,7 @@
 ![Matterbridge Logo](img/matterbridge-notext.gif)<br />
    **A simple chat bridge**<br />
    Letting people be where they want to be.<br />
-   <sub>Bridges between a growing number of protocols. Click below to demo.</sub>
+   <sub>Bridges between a growing number of protocols. Click below to demo or join the development chat.</sub>
 
    <sup>
 
@@ -15,9 +15,12 @@
       [Matrix][mb-matrix] |
       [Slack][mb-slack] |
       [Mattermost][mb-mattermost] |
+      [Rocket.Chat][mb-rocketchat] |
       [XMPP][mb-xmpp] |
       [Twitch][mb-twitch] |
+      [WhatsApp][mb-whatsapp] |
       [Zulip][mb-zulip] |
+      [Telegram][mb-telegram] |
       And more...
    </sup>
 
@@ -34,8 +37,10 @@
 
 ### Table of Contents
  * [Features](https://github.com/42wim/matterbridge/wiki/Features)
+   * [Natively supported](#natively-supported)
+   * [3rd party via matterbridge api](#3rd-party-via-matterbridge-api)
    * [API](#API)
- * [Requirements](#requirements)
+ * [Chat with us](#chat-with-us)
  * [Screenshots](https://github.com/42wim/matterbridge/wiki/)
  * [Installing](#installing)
    * [Binaries](#binaries)
@@ -61,18 +66,8 @@
 * [Private groups](https://github.com/42wim/matterbridge/wiki/Features#private-groups)
 * [API](https://github.com/42wim/matterbridge/wiki/Features#api)
 
-### API
+### Natively supported
-The API is very basic at the moment.   
-More info and examples on the [wiki](https://github.com/42wim/matterbridge/wiki/Api).
 
-Used by at least 3 projects. Feel free to make a PR to add your project to this list.
-
-* [MatterLink](https://github.com/elytra/MatterLink) (Matterbridge link for Minecraft Server chat)
-* [pyCord](https://github.com/NikkyAI/pyCord) (crossplatform chatbot)
-* [Mattereddit](https://github.com/bonehurtingjuice/mattereddit) (Reddit chat support)
-
-## Requirements
-Accounts to one of the supported bridges
 * [Mattermost](https://github.com/mattermost/mattermost-server/) 4.x, 5.x
 * [IRC](http://www.mirc.com/servers.html)
 * [XMPP](https://xmpp.org)
@@ -86,21 +81,56 @@ Accounts to one of the supported bridges
 * [Steam](https://store.steampowered.com/)
 * [Twitch](https://twitch.tv)
 * [Ssh-chat](https://github.com/shazow/ssh-chat)
+* [WhatsApp](https://www.whatsapp.com/)
 * [Zulip](https://zulipchat.com)
 
+### 3rd party via matterbridge api
+* [Minecraft](https://github.com/elytra/MatterLink)
+* [Reddit](https://github.com/bonehurtingjuice/mattereddit)
+* [Facebook messenger](https://github.com/VictorNine/fbridge)
+* [Discourse](https://github.com/DeclanHoare/matterbabble)
+
+### API
+The API is very basic at the moment.   
+More info and examples on the [wiki](https://github.com/42wim/matterbridge/wiki/Api).
+
+Used by the projects below. Feel free to make a PR to add your project to this list.
+
+* [MatterLink](https://github.com/elytra/MatterLink) (Matterbridge link for Minecraft Server chat)
+* [pyCord](https://github.com/NikkyAI/pyCord) (crossplatform chatbot)
+* [Mattereddit](https://github.com/bonehurtingjuice/mattereddit) (Reddit chat support)
+* [fbridge](https://github.com/VictorNine/fbridge) (Facebook messenger support)
+* [matterbabble](https://github.com/DeclanHoare/matterbabble) (Discourse support)
+
+## Chat with us
+
+Questions or want to test on your favorite platform? Join below:
+
+* [Gitter][mb-gitter]
+* [IRC][mb-irc]
+* [Discord][mb-discord]
+* [Matrix][mb-matrix]
+* [Slack][mb-slack]
+* [Mattermost][mb-mattermost]
+* [Rocket.Chat][mb-rocketchat]
+* [XMPP][mb-xmpp]
+* [Twitch][mb-twitch]
+* [Zulip][mb-zulip]
+* [Telegram][mb-telegram]
+
 ## Screenshots
 See https://github.com/42wim/matterbridge/wiki
 
 ## Installing
 ### Binaries
-* Latest stable release [v1.12.2](https://github.com/42wim/matterbridge/releases/latest)
+* Latest stable release [v1.14.0](https://github.com/42wim/matterbridge/releases/latest)
 * Development releases (follows master) can be downloaded [here](https://dl.bintray.com/42wim/nightly/)
 
 ### Packages
 * [Overview](https://repology.org/metapackage/matterbridge/versions)
 
 ### Building
-Go 1.8+ is required. Make sure you have [Go](https://golang.org/doc/install) properly installed, including setting up your [GOPATH](https://golang.org/doc/code.html#GOPATH).
+Go 1.9+ is required. Make sure you have [Go](https://golang.org/doc/install) properly installed, including setting up your [GOPATH](https://golang.org/doc/code.html#GOPATH).
 
 After Go is setup, download matterbridge to your $GOPATH directory.
 
@@ -209,10 +239,6 @@ See [changelog.md](https://github.com/42wim/matterbridge/blob/master/changelog.m
 
 See [FAQ](https://github.com/42wim/matterbridge/wiki/FAQ)
 
-Want to tip ?
-* eth: 0xb3f9b5387c66ad6be892bcb7bbc67862f3abc16f
-* btc: 1N7cKHj5SfqBHBzDJ6kad4BzeqUBBS2zhs
-
 ## Related projects
 * [FOSSRIT/infrastructure - roles/matterbridge](https://github.com/FOSSRIT/infrastructure/tree/master/roles/matterbridge) (Ansible role used to automate deployments of Matterbridge)
 * [matterbridge autoconfig](https://github.com/patcon/matterbridge-autoconfig)
@@ -222,6 +248,9 @@ Want to tip ?
 * [matterlink](https://github.com/elytra/MatterLink)
 * [mattermost-plugin](https://github.com/matterbridge/mattermost-plugin) - Run matterbridge as a plugin in mattermost
 * [pyCord](https://github.com/NikkyAI/pyCord) (crossplatform chatbot)
+* [fbridge](https://github.com/VictorNine/fbridge) (Facebook messenger support)
+* [isla](https://github.com/alphachung/isla) (Bot for Discord-Telegram groups used alongside matterbridge)
+* [matterbabble](https://github.com/DeclanHoare/matterbabble) (Connect Discourse threads to Matterbridge)
 
 ## Articles
 * [matterbridge on kubernetes](https://medium.freecodecamp.org/using-kubernetes-to-deploy-a-chat-gateway-or-when-technology-works-like-its-supposed-to-a169a8cd69a3)
@@ -232,6 +261,7 @@ Want to tip ?
 * https://www.algoo.fr/blog/2018/01/19/recouvrez-votre-liberte-en-quittant-slack-pour-un-mattermost-auto-heberge/
 * https://kopano.com/blog/matterbridge-bridging-mattermost-chat/
 * https://www.stitcher.com/s/?eid=52382713
+* https://daniele.tech/2019/02/how-to-use-matterbridge-to-connect-2-different-slack-workspaces/
 
 ## Thanks
 [![Digitalocean](https://snag.gy/3LVifX.jpg)](https://www.digitalocean.com/) for sponsoring demo/testing droplets.
@@ -245,11 +275,14 @@ Matterbridge wouldn't exist without these libraries:
 * irc - https://github.com/lrstanley/girc
 * mattermost - https://github.com/mattermost/mattermost-server
 * matrix - https://github.com/matrix-org/gomatrix
+* sshchat - https://github.com/shazow/ssh-chat
 * slack - https://github.com/nlopes/slack
 * steam - https://github.com/Philipp15b/go-steam
 * telegram - https://github.com/go-telegram-bot-api/telegram-bot-api
 * xmpp - https://github.com/mattn/go-xmpp
+* whatsapp - https://github.com/Rhymen/go-whatsapp/
 * zulip - https://github.com/ifo/gozulipbot
+* tengo - https://github.com/d5/tengo
 
 <!-- Links -->
 
@@ -259,6 +292,9 @@ Matterbridge wouldn't exist without these libraries:
    [mb-matrix]: https://riot.im/app/#/room/#matterbridge:matrix.org
    [mb-slack]: https://join.slack.com/matterbridgechat/shared_invite/MjEwODMxNjU1NDMwLTE0OTk2MTU3NTMtMzZkZmRiNDZhOA
    [mb-mattermost]: https://framateam.org/signup_user_complete/?id=tfqm33ggop8x3qgu4boeieta6e
+   [mb-rocketchat]: https://open.rocket.chat/channel/matterbridge
    [mb-xmpp]: https://inverse.chat/
    [mb-twitch]: https://www.twitch.tv/matterbridge
+   [mb-whatsapp]: https://www.whatsapp.com/
    [mb-zulip]: https://matterbridge.zulipchat.com/register/
+   [mb-telegram]: https://t.me/Matterbridge

bridge/bridge.go

@@ -2,10 +2,10 @@ package bridge
 
 import (
 	"strings"
+	"sync"
 
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/sirupsen/logrus"
-	"sync"
 )
 
 type Bridger interface {
@@ -17,6 +17,8 @@ type Bridger interface {
 
 type Bridge struct {
 	Bridger
+	*sync.RWMutex
+
 	Name           string
 	Account        string
 	Protocol       string
@@ -26,37 +28,34 @@ type Bridge struct {
 	Log            *logrus.Entry
 	Config         config.Config
 	General        *config.Protocol
-	*sync.RWMutex
 }
 
 type Config struct {
-	//	General *config.Protocol
-	Remote chan config.Message
-	Log    *logrus.Entry
 	*Bridge
+
+	Remote chan config.Message
 }
 
 // Factory is the factory function to create a bridge
 type Factory func(*Config) Bridger
 
 func New(bridge *config.Bridge) *Bridge {
-	b := &Bridge{
-		Channels: make(map[string]config.ChannelInfo),
-		RWMutex:  new(sync.RWMutex),
-		Joined:   make(map[string]bool),
-	}
 	accInfo := strings.Split(bridge.Account, ".")
 	protocol := accInfo[0]
 	name := accInfo[1]
-	b.Name = name
+
-	b.Protocol = protocol
+	return &Bridge{
-	b.Account = bridge.Account
+		RWMutex:  new(sync.RWMutex),
-	return b
+		Channels: make(map[string]config.ChannelInfo),
+		Name:     name,
+		Protocol: protocol,
+		Account:  bridge.Account,
+		Joined:   make(map[string]bool),
+	}
 }
 
 func (b *Bridge) JoinChannels() error {
-	err := b.joinChannels(b.Channels, b.Joined)
+	return b.joinChannels(b.Channels, b.Joined)
-	return err
 }
 
 // SetChannelMembers sets the newMembers to the bridge ChannelMembers

bridge/config/config.go

@@ -8,7 +8,6 @@ import (
 	"time"
 
 	"github.com/fsnotify/fsnotify"
-	prefixed "github.com/matterbridge/logrus-prefixed-formatter"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/viper"
 )
@@ -94,6 +93,7 @@ type Protocol struct {
 	MediaDownloadSize      int    // all protocols
 	MediaServerDownload    string
 	MediaServerUpload      string
+	MediaConvertWebPToPNG  bool       // telegram
 	MessageDelay           int        // IRC, time in millisecond to wait between messages
 	MessageFormat          string     // telegram
 	MessageLength          int        // IRC, max length of a message allowed
@@ -129,6 +129,7 @@ type Protocol struct {
 	SkipTLSVerify          bool       // IRC, mattermost
 	StripNick              bool       // all protocols
 	SyncTopic              bool       // slack
+	TengoModifyMessage     string     // general
 	Team                   string     // mattermost
 	Token                  string     // gitter, slack, discord, api
 	Topic                  string     // zulip
@@ -136,6 +137,7 @@ type Protocol struct {
 	UseAPI                 bool       // mattermost, slack
 	UseSASL                bool       // IRC
 	UseTLS                 bool       // IRC
+	UseDiscriminator       bool       // discord
 	UseFirstName           bool       // telegram
 	UseUserName            bool       // discord
 	UseInsecureURL         bool       // telegram
@@ -146,6 +148,7 @@ type Protocol struct {
 type ChannelOptions struct {
 	Key        string // irc, xmpp
 	WebhookURL string // discord
+	Topic      string // zulip
 }
 
 type Bridge struct {
@@ -184,6 +187,7 @@ type BridgeValues struct {
 	Telegram           map[string]Protocol
 	Rocketchat         map[string]Protocol
 	SSHChat            map[string]Protocol
+	WhatsApp           map[string]Protocol // TODO is this struct used? Search for "SlackLegacy" for example didn't return any results
 	Zulip              map[string]Protocol
 	General            Protocol
 	Gateway            []Gateway
@@ -200,63 +204,58 @@ type Config interface {
 }
 
 type config struct {
-	v *viper.Viper
 	sync.RWMutex
 
-	cv *BridgeValues
+	logger *logrus.Entry
+	v      *viper.Viper
+	cv     *BridgeValues
 }
 
-func NewConfig(cfgfile string) Config {
+// NewConfig instantiates a new configuration based on the specified configuration file path.
-	logrus.SetFormatter(&prefixed.TextFormatter{PrefixPadding: 13, DisableColors: true, FullTimestamp: false})
+func NewConfig(rootLogger *logrus.Logger, cfgfile string) Config {
-	flog := logrus.WithFields(logrus.Fields{"prefix": "config"})
+	logger := rootLogger.WithFields(logrus.Fields{"prefix": "config"})
+
 	viper.SetConfigFile(cfgfile)
-	input, err := getFileContents(cfgfile)
+	input, err := ioutil.ReadFile(cfgfile)
 	if err != nil {
-		logrus.Fatal(err)
+		logger.Fatalf("Failed to read configuration file: %#v", err)
 	}
-	mycfg := newConfigFromString(input)
+
+	mycfg := newConfigFromString(logger, input)
 	if mycfg.cv.General.MediaDownloadSize == 0 {
 		mycfg.cv.General.MediaDownloadSize = 1000000
 	}
 	viper.WatchConfig()
 	viper.OnConfigChange(func(e fsnotify.Event) {
-		flog.Println("Config file changed:", e.Name)
+		logger.Println("Config file changed:", e.Name)
 	})
 	return mycfg
 }
 
-func getFileContents(filename string) ([]byte, error) {
+// NewConfigFromString instantiates a new configuration based on the specified string.
-	input, err := ioutil.ReadFile(filename)
+func NewConfigFromString(rootLogger *logrus.Logger, input []byte) Config {
-	if err != nil {
+	logger := rootLogger.WithFields(logrus.Fields{"prefix": "config"})
-		logrus.Fatal(err)
+	return newConfigFromString(logger, input)
-		return []byte(nil), err
-	}
-	return input, nil
 }
 
-func NewConfigFromString(input []byte) Config {
+func newConfigFromString(logger *logrus.Entry, input []byte) *config {
-	return newConfigFromString(input)
-}
-
-func newConfigFromString(input []byte) *config {
 	viper.SetConfigType("toml")
 	viper.SetEnvPrefix("matterbridge")
-	viper.AddConfigPath(".")
 	viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_", "-", "_"))
 	viper.AutomaticEnv()
-	err := viper.ReadConfig(bytes.NewBuffer(input))
+
-	if err != nil {
+	if err := viper.ReadConfig(bytes.NewBuffer(input)); err != nil {
-		logrus.Fatal(err)
+		logger.Fatalf("Failed to parse the configuration: %#v", err)
 	}
 
 	cfg := &BridgeValues{}
-	err = viper.Unmarshal(cfg)
+	if err := viper.Unmarshal(cfg); err != nil {
-	if err != nil {
+		logger.Fatalf("Failed to load the configuration: %#v", err)
-		logrus.Fatal(err)
 	}
 	return &config{
-		v:  viper.GetViper(),
+		logger: logger,
-		cv: cfg,
+		v:      viper.GetViper(),
+		cv:     cfg,
 	}
 }
 
@@ -267,46 +266,44 @@ func (c *config) BridgeValues() *BridgeValues {
 func (c *config) GetBool(key string) (bool, bool) {
 	c.RLock()
 	defer c.RUnlock()
-	//	log.Debugf("getting bool %s = %#v", key, c.v.GetBool(key))
 	return c.v.GetBool(key), c.v.IsSet(key)
 }
 
 func (c *config) GetInt(key string) (int, bool) {
 	c.RLock()
 	defer c.RUnlock()
-	//	log.Debugf("getting int %s = %d", key, c.v.GetInt(key))
 	return c.v.GetInt(key), c.v.IsSet(key)
 }
 
 func (c *config) GetString(key string) (string, bool) {
 	c.RLock()
 	defer c.RUnlock()
-	//	log.Debugf("getting String %s = %s", key, c.v.GetString(key))
 	return c.v.GetString(key), c.v.IsSet(key)
 }
 
 func (c *config) GetStringSlice(key string) ([]string, bool) {
 	c.RLock()
 	defer c.RUnlock()
-	// log.Debugf("getting StringSlice %s = %#v", key, c.v.GetStringSlice(key))
 	return c.v.GetStringSlice(key), c.v.IsSet(key)
 }
 
 func (c *config) GetStringSlice2D(key string) ([][]string, bool) {
 	c.RLock()
 	defer c.RUnlock()
-	result := [][]string{}
+
-	if res, ok := c.v.Get(key).([]interface{}); ok {
+	res, ok := c.v.Get(key).([]interface{})
-		for _, entry := range res {
+	if !ok {
-			result2 := []string{}
+		return nil, false
-			for _, entry2 := range entry.([]interface{}) {
-				result2 = append(result2, entry2.(string))
-			}
-			result = append(result, result2)
-		}
-		return result, true
 	}
-	return result, false
+	var result [][]string
+	for _, entry := range res {
+		result2 := []string{}
+		for _, entry2 := range entry.([]interface{}) {
+			result2 = append(result2, entry2.(string))
+		}
+		result = append(result, result2)
+	}
+	return result, true
 }
 
 func GetIconURL(msg *Message, iconURL string) string {

bridge/discord/discord.go

@@ -75,6 +75,8 @@ func (b *Bdiscord) Connect() error {
 	b.c.AddHandler(b.memberUpdate)
 	b.c.AddHandler(b.messageUpdate)
 	b.c.AddHandler(b.messageDelete)
+	b.c.AddHandler(b.memberAdd)
+	b.c.AddHandler(b.memberRemove)
 	err = b.c.Open()
 	if err != nil {
 		return err
@@ -244,7 +246,7 @@ func (b *Bdiscord) Send(msg config.Message) (string, error) {
 			b.Log.Debugf("Setting webhook channel to \"%s\"", msg.Channel)
 			_, err := b.c.WebhookEdit(wID, "", "", channelID)
 			if err != nil {
-				b.Log.Errorf("Could not set webhook channel: %v", err)
+				b.Log.Errorf("Could not set webhook channel: %s", err)
 				return "", err
 			}
 		}
@@ -276,7 +278,7 @@ func (b *Bdiscord) Send(msg config.Message) (string, error) {
 		for _, rmsg := range helper.HandleExtra(&msg, b.General) {
 			rmsg.Text = helper.ClipMessage(rmsg.Text, MessageLength)
 			if _, err := b.c.ChannelMessageSend(channelID, rmsg.Username+rmsg.Text); err != nil {
-				b.Log.Errorf("Could not send message %#v: %v", rmsg, err)
+				b.Log.Errorf("Could not send message %#v: %s", rmsg, err)
 			}
 		}
 		// check if we have files to upload (from slack, telegram or mattermost)
@@ -358,7 +360,7 @@ func (b *Bdiscord) handleUploadFile(msg *config.Message, channelID string) (stri
 		}
 		_, err = b.c.ChannelMessageSendComplex(channelID, &m)
 		if err != nil {
-			return "", fmt.Errorf("file upload failed: %#v", err)
+			return "", fmt.Errorf("file upload failed: %s", err)
 		}
 	}
 	return "", nil

bridge/discord/handlers.go

@@ -36,7 +36,7 @@ func (b *Bdiscord) messageCreate(s *discordgo.Session, m *discordgo.MessageCreat
 		return
 	}
 	// if using webhooks, do not relay if it's ours
-	if b.useWebhook() && m.Author.Bot && b.isWebhookID(m.Author.ID) {
+	if b.useWebhook() && m.Author.Bot { // && b.isWebhookID(m.Author.ID) {
 		return
 	}
 
@@ -71,6 +71,9 @@ func (b *Bdiscord) messageCreate(s *discordgo.Session, m *discordgo.MessageCreat
 		rmsg.Username = b.getNick(m.Author)
 	} else {
 		rmsg.Username = m.Author.Username
+		if b.GetBool("UseDiscriminator") {
+			rmsg.Username += "#" + m.Author.Discriminator
+		}
 	}
 
 	// if we have embedded content add it to text
@@ -123,3 +126,45 @@ func (b *Bdiscord) memberUpdate(s *discordgo.Session, m *discordgo.GuildMemberUp
 		b.nickMemberMap[m.Member.Nick] = m.Member
 	}
 }
+
+func (b *Bdiscord) memberAdd(s *discordgo.Session, m *discordgo.GuildMemberAdd) {
+	if m.Member == nil {
+		b.Log.Warnf("Received member update with no member information: %#v", m)
+		return
+	}
+	username := m.Member.User.Username
+	if m.Member.Nick != "" {
+		username = m.Member.Nick
+	}
+
+	rmsg := config.Message{
+		Account:  b.Account,
+		Event:    config.EventJoinLeave,
+		Username: "system",
+		Text:     username + " joins",
+	}
+	b.Log.Debugf("<= Sending message from %s to gateway", b.Account)
+	b.Log.Debugf("<= Message is %#v", rmsg)
+	b.Remote <- rmsg
+}
+
+func (b *Bdiscord) memberRemove(s *discordgo.Session, m *discordgo.GuildMemberRemove) {
+	if m.Member == nil {
+		b.Log.Warnf("Received member update with no member information: %#v", m)
+		return
+	}
+	username := m.Member.User.Username
+	if m.Member.Nick != "" {
+		username = m.Member.Nick
+	}
+
+	rmsg := config.Message{
+		Account:  b.Account,
+		Event:    config.EventJoinLeave,
+		Username: "system",
+		Text:     username + " leaves",
+	}
+	b.Log.Debugf("<= Sending message from %s to gateway", b.Account)
+	b.Log.Debugf("<= Message is %#v", rmsg)
+	b.Remote <- rmsg
+}

bridge/discord/helpers.go

@@ -25,7 +25,7 @@ func (b *Bdiscord) getNick(user *discordgo.User) string {
 	// If we didn't find nick, search for it.
 	member, err := b.c.GuildMember(b.guildID, user.ID)
 	if err != nil {
-		b.Log.Warnf("Failed to fetch information for member %#v: %#v", user, err)
+		b.Log.Warnf("Failed to fetch information for member %#v: %s", user, err)
 		return user.Username
 	} else if member == nil {
 		b.Log.Warnf("Got no information for member %#v", user)

bridge/helper/helper.go

@@ -3,6 +3,7 @@ package helper
 import (
 	"bytes"
 	"fmt"
+	"image/png"
 	"io"
 	"net/http"
 	"regexp"
@@ -10,15 +11,19 @@ import (
 	"time"
 	"unicode/utf8"
 
+	"golang.org/x/image/webp"
+
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/sirupsen/logrus"
 	"gitlab.com/golang-commonmark/markdown"
 )
 
+// DownloadFile downloads the given non-authenticated URL.
 func DownloadFile(url string) (*[]byte, error) {
 	return DownloadFileAuth(url, "")
 }
 
+// DownloadFileAuth downloads the given URL using the specified authentication token.
 func DownloadFileAuth(url string, auth string) (*[]byte, error) {
 	var buf bytes.Buffer
 	client := &http.Client{
@@ -42,8 +47,8 @@ func DownloadFileAuth(url string, auth string) (*[]byte, error) {
 }
 
 // GetSubLines splits messages in newline-delimited lines. If maxLineLength is
-// specified as non-zero GetSubLines will and also clip long lines to the
+// specified as non-zero GetSubLines will also clip long lines to the maximum
-// maximum length and insert a warning marker that the line was clipped.
+// length and insert a warning marker that the line was clipped.
 //
 // TODO: The current implementation has the inconvenient that it disregards
 // word boundaries when splitting but this is hard to solve without potentially
@@ -79,18 +84,24 @@ func GetSubLines(message string, maxLineLength int) []string {
 	return lines
 }
 
-// handle all the stuff we put into extra
+// HandleExtra manages the supplementary details stored inside a message's 'Extra' field map.
 func HandleExtra(msg *config.Message, general *config.Protocol) []config.Message {
 	extra := msg.Extra
 	rmsg := []config.Message{}
 	for _, f := range extra[config.EventFileFailureSize] {
 		fi := f.(config.FileInfo)
 		text := fmt.Sprintf("file %s too big to download (%#v > allowed size: %#v)", fi.Name, fi.Size, general.MediaDownloadSize)
-		rmsg = append(rmsg, config.Message{Text: text, Username: "<system> ", Channel: msg.Channel, Account: msg.Account})
+		rmsg = append(rmsg, config.Message{
+			Text:     text,
+			Username: "<system> ",
+			Channel:  msg.Channel,
+			Account:  msg.Account,
+		})
 	}
 	return rmsg
 }
 
+// GetAvatar constructs a URL for a given user-avatar if it is available in the cache.
 func GetAvatar(av map[string]string, userid string, general *config.Protocol) string {
 	if sha, ok := av[userid]; ok {
 		return general.MediaServerDownload + "/" + sha + "/" + userid + ".png"
@@ -98,13 +109,15 @@ func GetAvatar(av map[string]string, userid string, general *config.Protocol) st
 	return ""
 }
 
-func HandleDownloadSize(flog *logrus.Entry, msg *config.Message, name string, size int64, general *config.Protocol) error {
+// HandleDownloadSize checks a specified filename against the configured download blacklist
+// and checks a specified file-size against the configure limit.
+func HandleDownloadSize(logger *logrus.Entry, msg *config.Message, name string, size int64, general *config.Protocol) error {
 	// check blacklist here
 	for _, entry := range general.MediaDownloadBlackList {
 		if entry != "" {
 			re, err := regexp.Compile(entry)
 			if err != nil {
-				flog.Errorf("incorrect regexp %s for %s", entry, msg.Account)
+				logger.Errorf("incorrect regexp %s for %s", entry, msg.Account)
 				continue
 			}
 			if re.MatchString(name) {
@@ -112,48 +125,77 @@ func HandleDownloadSize(flog *logrus.Entry, msg *config.Message, name string, si
 			}
 		}
 	}
-	flog.Debugf("Trying to download %#v with size %#v", name, size)
+	logger.Debugf("Trying to download %#v with size %#v", name, size)
 	if int(size) > general.MediaDownloadSize {
 		msg.Event = config.EventFileFailureSize
-		msg.Extra[msg.Event] = append(msg.Extra[msg.Event], config.FileInfo{Name: name, Comment: msg.Text, Size: size})
+		msg.Extra[msg.Event] = append(msg.Extra[msg.Event], config.FileInfo{
+			Name:    name,
+			Comment: msg.Text,
+			Size:    size,
+		})
 		return fmt.Errorf("File %#v to large to download (%#v). MediaDownloadSize is %#v", name, size, general.MediaDownloadSize)
 	}
 	return nil
 }
 
-func HandleDownloadData(flog *logrus.Entry, msg *config.Message, name, comment, url string, data *[]byte, general *config.Protocol) {
+// HandleDownloadData adds the data for a remote file into a Matterbridge gateway message.
+func HandleDownloadData(logger *logrus.Entry, msg *config.Message, name, comment, url string, data *[]byte, general *config.Protocol) {
 	var avatar bool
-	flog.Debugf("Download OK %#v %#v", name, len(*data))
+	logger.Debugf("Download OK %#v %#v", name, len(*data))
 	if msg.Event == config.EventAvatarDownload {
 		avatar = true
 	}
-	msg.Extra["file"] = append(msg.Extra["file"], config.FileInfo{Name: name, Data: data, URL: url, Comment: comment, Avatar: avatar})
+	msg.Extra["file"] = append(msg.Extra["file"], config.FileInfo{
+		Name:    name,
+		Data:    data,
+		URL:     url,
+		Comment: comment,
+		Avatar:  avatar,
+	})
 }
 
+var emptyLineMatcher = regexp.MustCompile("\n+")
+
+// RemoveEmptyNewLines collapses consecutive newline characters into a single one and
+// trims any preceding or trailing newline characters as well.
 func RemoveEmptyNewLines(msg string) string {
-	lines := ""
+	return emptyLineMatcher.ReplaceAllString(strings.Trim(msg, "\n"), "\n")
-	for _, line := range strings.Split(msg, "\n") {
-		if line != "" {
-			lines += line + "\n"
-		}
-	}
-	lines = strings.TrimRight(lines, "\n")
-	return lines
 }
 
+// ClipMessage trims a message to the specified length if it exceeds it and adds a warning
+// to the message in case it does so.
 func ClipMessage(text string, length int) string {
-	// clip too long messages
+	const clippingMessage = " <clipped message>"
 	if len(text) > length {
-		text = text[:length-len(" *message clipped*")]
+		text = text[:length-len(clippingMessage)]
 		if r, size := utf8.DecodeLastRuneInString(text); r == utf8.RuneError {
 			text = text[:len(text)-size]
 		}
-		text += " *message clipped*"
+		text += clippingMessage
 	}
 	return text
 }
 
 func ParseMarkdown(input string) string {
 	md := markdown.New(markdown.XHTMLOutput(true), markdown.Breaks(true))
-	return (md.RenderToString([]byte(input)))
+	res := md.RenderToString([]byte(input))
+	res = strings.TrimPrefix(res, "<p>")
+	res = strings.TrimSuffix(res, "</p>\n")
+	return res
+}
+
+// ConvertWebPToPNG convert input data (which should be WebP format to PNG format)
+func ConvertWebPToPNG(data *[]byte) error {
+	r := bytes.NewReader(*data)
+	m, err := webp.Decode(r)
+	if err != nil {
+		return err
+	}
+	var output []byte
+	w := bytes.NewBuffer(output)
+	if err := png.Encode(w, m); err != nil {
+		return err
+	}
+	*data = w.Bytes()
+	return nil
 }

bridge/helper/helper_test.go

@@ -1,6 +1,8 @@
 package helper
 
 import (
+	"io/ioutil"
+	"os"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -103,3 +105,22 @@ func TestGetSubLines(t *testing.T) {
 		assert.Equalf(t, testcase.nonSplitOutput, nonSplitLines, "'%s' testcase should give expected lines without splitting.", testname)
 	}
 }
+
+func TestConvertWebPToPNG(t *testing.T) {
+	if os.Getenv("LOCAL_TEST") == "" {
+		t.Skip()
+	}
+	input, err := ioutil.ReadFile("test.webp")
+	if err != nil {
+		t.Fail()
+	}
+	d := &input
+	err = ConvertWebPToPNG(d)
+	if err != nil {
+		t.Fail()
+	}
+	err = ioutil.WriteFile("test.png", *d, 0644)
+	if err != nil {
+		t.Fail()
+	}
+}

bridge/irc/handlers.go

@@ -81,7 +81,7 @@ func (b *Birc) handleJoinPart(client *girc.Client, event girc.Event) {
 		return
 	}
 	if event.Command == "QUIT" {
-		if event.Source.Name == b.Nick && strings.Contains(event.Trailing, "Ping timeout") {
+		if event.Source.Name == b.Nick && strings.Contains(event.Last(), "Ping timeout") {
 			b.Log.Infof("%s reconnecting ..", b.Account)
 			b.Remote <- config.Message{Username: "system", Text: "reconnect", Channel: channel, Account: b.Account, Event: config.EventFailure}
 			return
@@ -92,6 +92,10 @@ func (b *Birc) handleJoinPart(client *girc.Client, event girc.Event) {
 			return
 		}
 		b.Log.Debugf("<= Sending JOIN_LEAVE event from %s to gateway", b.Account)
+		// QUIT isn't channel bound, happens for all channels on the bridge
+		if event.Command == "QUIT" {
+			channel = ""
+		}
 		msg := config.Message{Username: "system", Text: event.Source.Name + " " + strings.ToLower(event.Command) + "s", Channel: channel, Account: b.Account, Event: config.EventJoinLeave}
 		b.Log.Debugf("<= Message is %#v", msg)
 		b.Remote <- msg
@@ -164,7 +168,7 @@ func (b *Birc) handlePrivMsg(client *girc.Client, event girc.Event) {
 		return
 	}
 	rmsg := config.Message{Username: event.Source.Name, Channel: strings.ToLower(event.Params[0]), Account: b.Account, UserID: event.Source.Ident + "@" + event.Source.Host}
-	b.Log.Debugf("== Receiving PRIVMSG: %s %s %#v", event.Source.Name, event.Trailing, event)
+	b.Log.Debugf("== Receiving PRIVMSG: %s %s %#v", event.Source.Name, event.Last(), event)
 
 	// set action event
 	if event.IsAction() {

bridge/irc/irc.go

@@ -295,7 +295,7 @@ func (b *Birc) storeNames(client *girc.Client, event girc.Event) {
 	channel := event.Params[2]
 	b.names[channel] = append(
 		b.names[channel],
-		strings.Split(strings.TrimSpace(event.Trailing), " ")...)
+		strings.Split(strings.TrimSpace(event.Last()), " ")...)
 }
 
 func (b *Birc) formatnicks(nicks []string) string {

bridge/matrix/matrix.go

@@ -20,11 +20,13 @@ type Bmatrix struct {
 	UserID  string
 	RoomMap map[string]string
 	sync.RWMutex
+	htmlTag *regexp.Regexp
 	*bridge.Config
 }
 
 func New(cfg *bridge.Config) bridge.Bridger {
 	b := &Bmatrix{Config: cfg}
+	b.htmlTag = regexp.MustCompile("</.*?>")
 	b.RoomMap = make(map[string]string)
 	return b
 }
@@ -113,8 +115,22 @@ func (b *Bmatrix) Send(msg config.Message) (string, error) {
 	// Edit message if we have an ID
 	// matrix has no editing support
 
+	// Use notices to send join/leave events
+	if msg.Event == config.EventJoinLeave {
+		resp, err := b.mc.SendNotice(channel, msg.Username+msg.Text)
+		if err != nil {
+			return "", err
+		}
+		return resp.EventID, err
+	}
+
+	username := html.EscapeString(msg.Username)
+	// check if we have a </tag>. if we have, we don't escape HTML. #696
+	if b.htmlTag.MatchString(msg.Username) {
+		username = msg.Username
+	}
 	// Post normal message with HTML support (eg riot.im)
-	resp, err := b.mc.SendHTML(channel, msg.Username+msg.Text, html.EscapeString(msg.Username)+helper.ParseMarkdown(msg.Text))
+	resp, err := b.mc.SendHTML(channel, msg.Username+msg.Text, username+helper.ParseMarkdown(msg.Text))
 	if err != nil {
 		return "", err
 	}
@@ -283,6 +299,12 @@ func (b *Bmatrix) handleUploadFile(msg *config.Message, channel string, fi *conf
 		if err != nil {
 			b.Log.Errorf("file comment failed: %#v", err)
 		}
+	} else {
+		// image and video uploads send no username, we have to do this ourself here #715
+		_, err := b.mc.SendText(channel, msg.Username)
+		if err != nil {
+			b.Log.Errorf("file comment failed: %#v", err)
+		}
 	}
 	b.Log.Debugf("uploading file: %s %s", fi.Name, mtype)
 	res, err := b.mc.UploadToContentRepo(content, mtype, int64(len(*fi.Data)))

bridge/rocketchat/handlers.go

@@ -0,0 +1,74 @@
+package brocketchat
+
+import (
+	"github.com/42wim/matterbridge/bridge/config"
+)
+
+func (b *Brocketchat) handleRocket() {
+	messages := make(chan *config.Message)
+	if b.GetString("WebhookBindAddress") != "" {
+		b.Log.Debugf("Choosing webhooks based receiving")
+		go b.handleRocketHook(messages)
+	} else {
+		b.Log.Debugf("Choosing login/password based receiving")
+		go b.handleRocketClient(messages)
+	}
+	for message := range messages {
+		message.Account = b.Account
+		b.Log.Debugf("<= Sending message from %s on %s to gateway", message.Username, b.Account)
+		b.Log.Debugf("<= Message is %#v", message)
+		b.Remote <- *message
+	}
+}
+
+func (b *Brocketchat) handleRocketHook(messages chan *config.Message) {
+	for {
+		message := b.rh.Receive()
+		b.Log.Debugf("Receiving from rockethook %#v", message)
+		// do not loop
+		if message.UserName == b.GetString("Nick") {
+			continue
+		}
+		messages <- &config.Message{
+			UserID:   message.UserID,
+			Username: message.UserName,
+			Text:     message.Text,
+			Channel:  message.ChannelName,
+		}
+	}
+}
+
+func (b *Brocketchat) handleRocketClient(messages chan *config.Message) {
+	for message := range b.messageChan {
+		// skip messages with same ID, apparently messages get duplicated for an unknown reason
+		if _, ok := b.cache.Get(message.ID); ok {
+			continue
+		}
+		b.cache.Add(message.ID, true)
+		b.Log.Debugf("message %#v", message)
+		m := message
+		if b.skipMessage(&m) {
+			b.Log.Debugf("Skipped message: %#v", message)
+			continue
+		}
+
+		rmsg := &config.Message{Text: message.Msg,
+			Username: message.User.UserName,
+			Channel:  b.getChannelName(message.RoomID),
+			Account:  b.Account,
+			UserID:   message.User.ID,
+			ID:       message.ID,
+		}
+		messages <- rmsg
+	}
+}
+
+func (b *Brocketchat) handleUploadFile(msg *config.Message) error {
+	for _, f := range msg.Extra["file"] {
+		fi := f.(config.FileInfo)
+		if err := b.uploadFile(&fi, b.getChannelID(msg.Channel)); err != nil {
+			return err
+		}
+	}
+	return nil
+}

bridge/rocketchat/helpers.go

@@ -0,0 +1,198 @@
+package brocketchat
+
+import (
+	"context"
+	"io/ioutil"
+	"mime"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/42wim/matterbridge/bridge/helper"
+	"github.com/42wim/matterbridge/hook/rockethook"
+	"github.com/42wim/matterbridge/matterhook"
+	"github.com/matterbridge/Rocket.Chat.Go.SDK/models"
+	"github.com/matterbridge/Rocket.Chat.Go.SDK/realtime"
+	"github.com/matterbridge/Rocket.Chat.Go.SDK/rest"
+	"github.com/nelsonken/gomf"
+)
+
+func (b *Brocketchat) doConnectWebhookBind() error {
+	switch {
+	case b.GetString("WebhookURL") != "":
+		b.Log.Info("Connecting using webhookurl (sending) and webhookbindaddress (receiving)")
+		b.mh = matterhook.New(b.GetString("WebhookURL"),
+			matterhook.Config{InsecureSkipVerify: b.GetBool("SkipTLSVerify"),
+				DisableServer: true})
+		b.rh = rockethook.New(b.GetString("WebhookURL"), rockethook.Config{BindAddress: b.GetString("WebhookBindAddress")})
+	case b.GetString("Login") != "":
+		b.Log.Info("Connecting using login/password (sending)")
+		err := b.apiLogin()
+		if err != nil {
+			return err
+		}
+	default:
+		b.Log.Info("Connecting using webhookbindaddress (receiving)")
+		b.rh = rockethook.New(b.GetString("WebhookURL"), rockethook.Config{BindAddress: b.GetString("WebhookBindAddress")})
+	}
+	return nil
+}
+
+func (b *Brocketchat) doConnectWebhookURL() error {
+	b.Log.Info("Connecting using webhookurl (sending)")
+	b.mh = matterhook.New(b.GetString("WebhookURL"),
+		matterhook.Config{InsecureSkipVerify: b.GetBool("SkipTLSVerify"),
+			DisableServer: true})
+	if b.GetString("Login") != "" {
+		b.Log.Info("Connecting using login/password (receiving)")
+		err := b.apiLogin()
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (b *Brocketchat) apiLogin() error {
+	b.Log.Debugf("handling apiLogin()")
+	credentials := &models.UserCredentials{Email: b.GetString("login"), Password: b.GetString("password")}
+	myURL, err := url.Parse(b.GetString("server"))
+	if err != nil {
+		return err
+	}
+	client, err := realtime.NewClient(myURL, b.GetBool("debug"))
+	b.c = client
+	if err != nil {
+		return err
+	}
+	restclient := rest.NewClient(myURL, b.GetBool("debug"))
+	user, err := b.c.Login(credentials)
+	if err != nil {
+		return err
+	}
+	b.user = user
+	b.r = restclient
+	err = b.r.Login(credentials)
+	if err != nil {
+		return err
+	}
+	b.Log.Info("Connection succeeded")
+	return nil
+}
+
+func (b *Brocketchat) getChannelName(id string) string {
+	b.RLock()
+	defer b.RUnlock()
+	if name, ok := b.channelMap[id]; ok {
+		return name
+	}
+	return ""
+}
+
+func (b *Brocketchat) getChannelID(name string) string {
+	b.RLock()
+	defer b.RUnlock()
+	for k, v := range b.channelMap {
+		if v == name {
+			return k
+		}
+	}
+	return ""
+}
+
+func (b *Brocketchat) skipMessage(message *models.Message) bool {
+	return message.User.ID == b.user.ID
+}
+
+func (b *Brocketchat) uploadFile(fi *config.FileInfo, channel string) error {
+	fb := gomf.New()
+	if err := fb.WriteField("description", fi.Comment); err != nil {
+		return err
+	}
+	sp := strings.Split(fi.Name, ".")
+	mtype := mime.TypeByExtension("." + sp[len(sp)-1])
+	if !strings.Contains(mtype, "image") && !strings.Contains(mtype, "video") {
+		return nil
+	}
+	if err := fb.WriteFile("file", fi.Name, mtype, *fi.Data); err != nil {
+		return err
+	}
+	req, err := fb.GetHTTPRequest(context.TODO(), b.GetString("server")+"/api/v1/rooms.upload/"+channel)
+	if err != nil {
+		return err
+	}
+	req.Header.Add("X-Auth-Token", b.user.Token)
+	req.Header.Add("X-User-Id", b.user.ID)
+	client := &http.Client{
+		Timeout: time.Second * 5,
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	if resp.StatusCode != 200 {
+		b.Log.Errorf("failed: %#v", string(body))
+	}
+	return nil
+}
+
+// sendWebhook uses the configured WebhookURL to send the message
+func (b *Brocketchat) sendWebhook(msg *config.Message) error {
+	// skip events
+	if msg.Event != "" {
+		return nil
+	}
+
+	if b.GetBool("PrefixMessagesWithNick") {
+		msg.Text = msg.Username + msg.Text
+	}
+	if msg.Extra != nil {
+		// this sends a message only if we received a config.EVENT_FILE_FAILURE_SIZE
+		for _, rmsg := range helper.HandleExtra(msg, b.General) {
+			rmsg := rmsg // scopelint
+			iconURL := config.GetIconURL(&rmsg, b.GetString("iconurl"))
+			matterMessage := matterhook.OMessage{
+				IconURL:  iconURL,
+				Channel:  rmsg.Channel,
+				UserName: rmsg.Username,
+				Text:     rmsg.Text,
+				Props:    make(map[string]interface{}),
+			}
+			if err := b.mh.Send(matterMessage); err != nil {
+				b.Log.Errorf("sendWebhook failed: %s ", err)
+			}
+		}
+
+		// webhook doesn't support file uploads, so we add the url manually
+		if len(msg.Extra["file"]) > 0 {
+			for _, f := range msg.Extra["file"] {
+				fi := f.(config.FileInfo)
+				if fi.URL != "" {
+					msg.Text += fi.URL
+				}
+			}
+		}
+	}
+	iconURL := config.GetIconURL(msg, b.GetString("iconurl"))
+	matterMessage := matterhook.OMessage{
+		IconURL:  iconURL,
+		Channel:  msg.Channel,
+		UserName: msg.Username,
+		Text:     msg.Text,
+	}
+	if msg.Avatar != "" {
+		matterMessage.IconURL = msg.Avatar
+	}
+	err := b.mh.Send(matterMessage)
+	if err != nil {
+		b.Log.Info(err)
+		return err
+	}
+	return nil
+}

bridge/rocketchat/rocketchat.go

@@ -1,21 +1,47 @@
 package brocketchat
 
 import (
+	"errors"
+	"strings"
+	"sync"
+
 	"github.com/42wim/matterbridge/bridge"
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/42wim/matterbridge/bridge/helper"
 	"github.com/42wim/matterbridge/hook/rockethook"
 	"github.com/42wim/matterbridge/matterhook"
+	lru "github.com/hashicorp/golang-lru"
+	"github.com/matterbridge/Rocket.Chat.Go.SDK/models"
+	"github.com/matterbridge/Rocket.Chat.Go.SDK/realtime"
+	"github.com/matterbridge/Rocket.Chat.Go.SDK/rest"
 )
 
 type Brocketchat struct {
-	mh *matterhook.Client
+	mh    *matterhook.Client
-	rh *rockethook.Client
+	rh    *rockethook.Client
+	c     *realtime.Client
+	r     *rest.Client
+	cache *lru.Cache
 	*bridge.Config
+	messageChan chan models.Message
+	channelMap  map[string]string
+	user        *models.User
+	sync.RWMutex
 }
 
 func New(cfg *bridge.Config) bridge.Bridger {
-	return &Brocketchat{Config: cfg}
+	newCache, err := lru.New(100)
+	if err != nil {
+		cfg.Log.Fatalf("Could not create LRU cache for rocketchat bridge: %v", err)
+	}
+	b := &Brocketchat{
+		Config:      cfg,
+		messageChan: make(chan models.Message),
+		channelMap:  make(map[string]string),
+		cache:       newCache,
+	}
+	b.Log.Debugf("enabling rocketchat")
+	return b
 }
 
 func (b *Brocketchat) Command(cmd string) string {
@@ -23,70 +49,122 @@ func (b *Brocketchat) Command(cmd string) string {
 }
 
 func (b *Brocketchat) Connect() error {
-	b.Log.Info("Connecting webhooks")
+	if b.GetString("WebhookBindAddress") != "" {
-	b.mh = matterhook.New(b.GetString("WebhookURL"),
+		if err := b.doConnectWebhookBind(); err != nil {
-		matterhook.Config{InsecureSkipVerify: b.GetBool("SkipTLSVerify"),
+			return err
-			DisableServer: true})
+		}
-	b.rh = rockethook.New(b.GetString("WebhookURL"), rockethook.Config{BindAddress: b.GetString("WebhookBindAddress")})
+		go b.handleRocket()
-	go b.handleRocketHook()
+		return nil
+	}
+	switch {
+	case b.GetString("WebhookURL") != "":
+		if err := b.doConnectWebhookURL(); err != nil {
+			return err
+		}
+		go b.handleRocket()
+		return nil
+	case b.GetString("Login") != "":
+		b.Log.Info("Connecting using login/password (sending and receiving)")
+		err := b.apiLogin()
+		if err != nil {
+			return err
+		}
+		go b.handleRocket()
+	}
+	if b.GetString("WebhookBindAddress") == "" && b.GetString("WebhookURL") == "" &&
+		b.GetString("Login") == "" {
+		return errors.New("no connection method found. See that you have WebhookBindAddress, WebhookURL or Login/Password/Server configured")
+	}
 	return nil
 }
 
 func (b *Brocketchat) Disconnect() error {
 	return nil
-
 }
 
 func (b *Brocketchat) JoinChannel(channel config.ChannelInfo) error {
+	if b.c == nil {
+		return nil
+	}
+	id, err := b.c.GetChannelId(strings.TrimPrefix(channel.Name, "#"))
+	if err != nil {
+		return err
+	}
+	b.Lock()
+	b.channelMap[id] = channel.Name
+	b.Unlock()
+	mychannel := &models.Channel{ID: id, Name: strings.TrimPrefix(channel.Name, "#")}
+	if err := b.c.JoinChannel(id); err != nil {
+		return err
+	}
+	if err := b.c.SubscribeToMessageStream(mychannel, b.messageChan); err != nil {
+		return err
+	}
 	return nil
 }
 
 func (b *Brocketchat) Send(msg config.Message) (string, error) {
-	// ignore delete messages
+	// strip the # if people has set this
+	msg.Channel = strings.TrimPrefix(msg.Channel, "#")
+	channel := &models.Channel{ID: b.getChannelID(msg.Channel), Name: msg.Channel}
+
+	// Delete message
 	if msg.Event == config.EventMsgDelete {
-		return "", nil
+		if msg.ID == "" {
+			return "", nil
+		}
+		return msg.ID, b.c.DeleteMessage(&models.Message{ID: msg.ID})
 	}
-	b.Log.Debugf("=> Receiving %#v", msg)
+
+	// Use webhook to send the message
+	if b.GetString("WebhookURL") != "" {
+		return "", b.sendWebhook(&msg)
+	}
+
+	// Prepend nick if configured
+	if b.GetBool("PrefixMessagesWithNick") {
+		msg.Text = msg.Username + msg.Text
+	}
+
+	// Edit message if we have an ID
+	if msg.ID != "" {
+		return msg.ID, b.c.EditMessage(&models.Message{ID: msg.ID, Msg: msg.Text, RoomID: b.getChannelID(msg.Channel)})
+	}
+
+	// Upload a file if it exists
 	if msg.Extra != nil {
 		for _, rmsg := range helper.HandleExtra(&msg, b.General) {
-			rmsg := rmsg // scopelint
+			// strip the # if people has set this
-			iconURL := config.GetIconURL(&rmsg, b.GetString("iconurl"))
+			rmsg.Channel = strings.TrimPrefix(rmsg.Channel, "#")
-			matterMessage := matterhook.OMessage{IconURL: iconURL, Channel: rmsg.Channel, UserName: rmsg.Username, Text: rmsg.Text}
+			smsg := &models.Message{
-			b.mh.Send(matterMessage)
+				RoomID: b.getChannelID(rmsg.Channel),
-		}
+				Msg:    rmsg.Username + rmsg.Text,
-		if len(msg.Extra["file"]) > 0 {
+				PostMessage: models.PostMessage{
-			for _, f := range msg.Extra["file"] {
+					Avatar: rmsg.Avatar,
-				fi := f.(config.FileInfo)
+					Alias:  rmsg.Username,
-				if fi.URL != "" {
+				},
-					msg.Text += fi.URL
+			}
-				}
+			if _, err := b.c.SendMessage(smsg); err != nil {
+				b.Log.Errorf("SendMessage failed: %s", err)
 			}
 		}
+		if len(msg.Extra["file"]) > 0 {
+			return "", b.handleUploadFile(&msg)
+		}
 	}
 
-	iconURL := config.GetIconURL(&msg, b.GetString("iconurl"))
+	smsg := &models.Message{
-	matterMessage := matterhook.OMessage{IconURL: iconURL}
+		RoomID: channel.ID,
-	matterMessage.Channel = msg.Channel
+		Msg:    msg.Text,
-	matterMessage.UserName = msg.Username
+		PostMessage: models.PostMessage{
-	matterMessage.Type = ""
+			Avatar: msg.Avatar,
-	matterMessage.Text = msg.Text
+			Alias:  msg.Username,
-	err := b.mh.Send(matterMessage)
+		},
-	if err != nil {
+	}
-		b.Log.Info(err)
+
+	rmsg, err := b.c.SendMessage(smsg)
+	if rmsg == nil {
 		return "", err
 	}
-	return "", nil
+	return rmsg.ID, err
-}
-
-func (b *Brocketchat) handleRocketHook() {
-	for {
-		message := b.rh.Receive()
-		b.Log.Debugf("Receiving from rockethook %#v", message)
-		// do not loop
-		if message.UserName == b.GetString("Nick") {
-			continue
-		}
-		b.Log.Debugf("<= Sending message from %s on %s to gateway", message.UserName, b.Account)
-		b.Remote <- config.Message{Text: message.Text, Username: message.UserName, Channel: message.ChannelName, Account: b.Account, UserID: message.UserID}
-	}
 }

bridge/slack/handlers.go

@@ -34,7 +34,7 @@ func (b *Bslack) handleSlack() {
 		message.Text = html.UnescapeString(message.Text)
 
 		// Add the avatar
-		message.Avatar = b.getAvatar(message.UserID)
+		message.Avatar = b.users.getAvatar(message.UserID)
 
 		b.Log.Debugf("<= Message is %#v", message)
 		b.Remote <- *message
@@ -75,20 +75,17 @@ func (b *Bslack) handleSlackClient(messages chan *config.Message) {
 			// When we join a channel we update the full list of users as
 			// well as the information for the channel that we joined as this
 			// should now tell that we are a member of it.
-			b.channelsMutex.Lock()
+			b.channels.registerChannel(ev.Channel)
-			b.channelsByID[ev.Channel.ID] = &ev.Channel
-			b.channelsByName[ev.Channel.Name] = &ev.Channel
-			b.channelsMutex.Unlock()
 		case *slack.ConnectedEvent:
 			b.si = ev.Info
-			b.populateChannels(true)
+			b.channels.populateChannels(true)
-			b.populateUsers(true)
+			b.users.populateUsers(true)
 		case *slack.InvalidAuthEvent:
 			b.Log.Fatalf("Invalid Token %#v", ev)
 		case *slack.ConnectionErrorEvent:
 			b.Log.Errorf("Connection failed %#v %#v", ev.Error(), ev.ErrorObj)
 		case *slack.MemberJoinedChannelEvent:
-			b.populateUser(ev.User)
+			b.users.populateUser(ev.User)
 		case *slack.LatencyReport:
 			continue
 		default:
@@ -192,6 +189,9 @@ func (b *Bslack) handleMessageEvent(ev *slack.MessageEvent) (*config.Message, er
 			// This is probably a webhook we couldn't resolve.
 			return nil, fmt.Errorf("message handling resulted in an empty bot message (probably an incoming webhook we couldn't resolve): %#v", ev)
 		}
+		if ev.SubMessage != nil {
+			return nil, fmt.Errorf("message handling resulted in an empty message: %#v with submessage %#v", ev, ev.SubMessage)
+		}
 		return nil, fmt.Errorf("message handling resulted in an empty message: %#v", ev)
 	}
 	return rmsg, nil
@@ -207,7 +207,7 @@ func (b *Bslack) handleStatusEvent(ev *slack.MessageEvent, rmsg *config.Message)
 		rmsg.Username = sSystemUser
 		rmsg.Event = config.EventJoinLeave
 	case sChannelTopic, sChannelPurpose:
-		b.populateChannels(false)
+		b.channels.populateChannels(false)
 		rmsg.Event = config.EventTopicChange
 	case sMessageChanged:
 		rmsg.Text = ev.SubMessage.Text
@@ -263,7 +263,7 @@ func (b *Bslack) handleAttachments(ev *slack.MessageEvent, rmsg *config.Message)
 }
 
 func (b *Bslack) handleTypingEvent(ev *slack.UserTypingEvent) (*config.Message, error) {
-	channelInfo, err := b.getChannelByID(ev.Channel)
+	channelInfo, err := b.channels.getChannelByID(ev.Channel)
 	if err != nil {
 		return nil, err
 	}
@@ -313,36 +313,7 @@ func (b *Bslack) handleGetChannelMembers(rmsg *config.Message) bool {
 		return false
 	}
 
-	cMembers := config.ChannelMembers{}
+	cMembers := b.channels.getChannelMembers(b.users)
-
-	b.channelMembersMutex.RLock()
-
-	for channelID, members := range b.channelMembers {
-		for _, member := range members {
-			channelName := ""
-			userName := ""
-			userNick := ""
-			user := b.getUser(member)
-			if user != nil {
-				userName = user.Name
-				userNick = user.Profile.DisplayName
-			}
-			channel, _ := b.getChannelByID(channelID)
-			if channel != nil {
-				channelName = channel.Name
-			}
-			cMember := config.ChannelMember{
-				Username:    userName,
-				Nick:        userNick,
-				UserID:      member,
-				ChannelID:   channelID,
-				ChannelName: channelName,
-			}
-			cMembers = append(cMembers, cMember)
-		}
-	}
-
-	b.channelMembersMutex.RUnlock()
 
 	extra := make(map[string][]interface{})
 	extra[config.EventGetChannelMembers] = append(extra[config.EventGetChannelMembers], cMembers)

bridge/slack/helpers.go

@@ -1,7 +1,6 @@
 package bslack
 
 import (
-	"context"
 	"fmt"
 	"regexp"
 	"strings"
@@ -9,225 +8,14 @@ import (
 
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/nlopes/slack"
+	"github.com/sirupsen/logrus"
 )
 
-func (b *Bslack) getUser(id string) *slack.User {
-	b.usersMutex.RLock()
-	user, ok := b.users[id]
-	b.usersMutex.RUnlock()
-	if ok {
-		return user
-	}
-	b.populateUser(id)
-	b.usersMutex.RLock()
-	defer b.usersMutex.RUnlock()
-
-	return b.users[id]
-}
-
-func (b *Bslack) getUsername(id string) string {
-	if user := b.getUser(id); user != nil {
-		if user.Profile.DisplayName != "" {
-			return user.Profile.DisplayName
-		}
-		return user.Name
-	}
-	b.Log.Warnf("Could not find user with ID '%s'", id)
-	return ""
-}
-
-func (b *Bslack) getAvatar(id string) string {
-	if user := b.getUser(id); user != nil {
-		return user.Profile.Image48
-	}
-	return ""
-}
-
-func (b *Bslack) getChannel(channel string) (*slack.Channel, error) {
-	if strings.HasPrefix(channel, "ID:") {
-		return b.getChannelByID(strings.TrimPrefix(channel, "ID:"))
-	}
-	return b.getChannelByName(channel)
-}
-
-func (b *Bslack) getChannelByName(name string) (*slack.Channel, error) {
-	return b.getChannelBy(name, b.channelsByName)
-}
-
-func (b *Bslack) getChannelByID(ID string) (*slack.Channel, error) {
-	return b.getChannelBy(ID, b.channelsByID)
-}
-
-func (b *Bslack) getChannelBy(lookupKey string, lookupMap map[string]*slack.Channel) (*slack.Channel, error) {
-	b.channelsMutex.RLock()
-	defer b.channelsMutex.RUnlock()
-
-	if channel, ok := lookupMap[lookupKey]; ok {
-		return channel, nil
-	}
-	return nil, fmt.Errorf("%s: channel %s not found", b.Account, lookupKey)
-}
-
-const minimumRefreshInterval = 10 * time.Second
-
-func (b *Bslack) populateUser(userID string) {
-	b.usersMutex.RLock()
-	_, exists := b.users[userID]
-	b.usersMutex.RUnlock()
-	if exists {
-		// already in cache
-		return
-	}
-
-	user, err := b.sc.GetUserInfo(userID)
-	if err != nil {
-		b.Log.Debugf("GetUserInfo failed for %v: %v", userID, err)
-		return
-	}
-
-	b.usersMutex.Lock()
-	b.users[userID] = user
-	b.usersMutex.Unlock()
-}
-
-func (b *Bslack) populateUsers(wait bool) {
-	b.refreshMutex.Lock()
-	if !wait && (time.Now().Before(b.earliestUserRefresh) || b.refreshInProgress) {
-		b.Log.Debugf("Not refreshing user list as it was done less than %v ago.",
-			minimumRefreshInterval)
-		b.refreshMutex.Unlock()
-
-		return
-	}
-	for b.refreshInProgress {
-		b.refreshMutex.Unlock()
-		time.Sleep(time.Second)
-		b.refreshMutex.Lock()
-	}
-	b.refreshInProgress = true
-	b.refreshMutex.Unlock()
-
-	newUsers := map[string]*slack.User{}
-	pagination := b.sc.GetUsersPaginated(slack.GetUsersOptionLimit(200))
-	count := 0
-	for {
-		var err error
-		pagination, err = pagination.Next(context.Background())
-		time.Sleep(time.Second)
-		if err != nil {
-			if pagination.Done(err) {
-				break
-			}
-
-			if err = b.handleRateLimit(err); err != nil {
-				b.Log.Errorf("Could not retrieve users: %#v", err)
-				return
-			}
-			continue
-		}
-
-		for i := range pagination.Users {
-			newUsers[pagination.Users[i].ID] = &pagination.Users[i]
-		}
-		b.Log.Debugf("getting %d users", len(pagination.Users))
-		count++
-		// more > 2000 users, slack will complain and ratelimit. break
-		if count > 10 {
-			b.Log.Info("Large slack detected > 2000 users, skipping loading complete userlist.")
-			break
-		}
-	}
-
-	b.usersMutex.Lock()
-	defer b.usersMutex.Unlock()
-	b.users = newUsers
-
-	b.refreshMutex.Lock()
-	defer b.refreshMutex.Unlock()
-	b.earliestUserRefresh = time.Now().Add(minimumRefreshInterval)
-	b.refreshInProgress = false
-}
-
-func (b *Bslack) populateChannels(wait bool) {
-	b.refreshMutex.Lock()
-	if !wait && (time.Now().Before(b.earliestChannelRefresh) || b.refreshInProgress) {
-		b.Log.Debugf("Not refreshing channel list as it was done less than %v seconds ago.",
-			minimumRefreshInterval)
-		b.refreshMutex.Unlock()
-		return
-	}
-	for b.refreshInProgress {
-		b.refreshMutex.Unlock()
-		time.Sleep(time.Second)
-		b.refreshMutex.Lock()
-	}
-	b.refreshInProgress = true
-	b.refreshMutex.Unlock()
-
-	newChannelsByID := map[string]*slack.Channel{}
-	newChannelsByName := map[string]*slack.Channel{}
-	newChannelMembers := make(map[string][]string)
-
-	// We only retrieve public and private channels, not IMs
-	// and MPIMs as those do not have a channel name.
-	queryParams := &slack.GetConversationsParameters{
-		ExcludeArchived: "true",
-		Types:           []string{"public_channel,private_channel"},
-	}
-	for {
-		channels, nextCursor, err := b.sc.GetConversations(queryParams)
-		if err != nil {
-			if err = b.handleRateLimit(err); err != nil {
-				b.Log.Errorf("Could not retrieve channels: %#v", err)
-				return
-			}
-			continue
-		}
-
-		for i := range channels {
-			newChannelsByID[channels[i].ID] = &channels[i]
-			newChannelsByName[channels[i].Name] = &channels[i]
-			// also find all the members in every channel
-			// comment for now, issues on big slacks
-			/*
-				members, err := b.getUsersInConversation(channels[i].ID)
-				if err != nil {
-					if err = b.handleRateLimit(err); err != nil {
-						b.Log.Errorf("Could not retrieve channel members: %#v", err)
-						return
-					}
-					continue
-				}
-				newChannelMembers[channels[i].ID] = members
-			*/
-		}
-
-		if nextCursor == "" {
-			break
-		}
-		queryParams.Cursor = nextCursor
-	}
-
-	b.channelsMutex.Lock()
-	defer b.channelsMutex.Unlock()
-	b.channelsByID = newChannelsByID
-	b.channelsByName = newChannelsByName
-
-	b.channelMembersMutex.Lock()
-	defer b.channelMembersMutex.Unlock()
-	b.channelMembers = newChannelMembers
-
-	b.refreshMutex.Lock()
-	defer b.refreshMutex.Unlock()
-	b.earliestChannelRefresh = time.Now().Add(minimumRefreshInterval)
-	b.refreshInProgress = false
-}
-
 // populateReceivedMessage shapes the initial Matterbridge message that we will forward to the
 // router before we apply message-dependent modifications.
 func (b *Bslack) populateReceivedMessage(ev *slack.MessageEvent) (*config.Message, error) {
 	// Use our own func because rtm.GetChannelInfo doesn't work for private channels.
-	channel, err := b.getChannelByID(ev.Channel)
+	channel, err := b.channels.getChannelByID(ev.Channel)
 	if err != nil {
 		return nil, err
 	}
@@ -254,6 +42,13 @@ func (b *Bslack) populateReceivedMessage(ev *slack.MessageEvent) (*config.Messag
 		}
 	}
 
+	// For edits, only submessage has thread ts.
+	// Ensures edits to threaded messages maintain their prefix hint on the
+	// unthreaded end.
+	if ev.SubMessage != nil {
+		rmsg.ParentID = ev.SubMessage.ThreadTimestamp
+	}
+
 	if err = b.populateMessageWithUserInfo(ev, rmsg); err != nil {
 		return nil, err
 	}
@@ -282,7 +77,7 @@ func (b *Bslack) populateMessageWithUserInfo(ev *slack.MessageEvent, rmsg *confi
 		return nil
 	}
 
-	user := b.getUser(userID)
+	user := b.users.getUser(userID)
 	if user == nil {
 		return fmt.Errorf("could not find information for user with id %s", ev.User)
 	}
@@ -308,7 +103,7 @@ func (b *Bslack) populateMessageWithBotInfo(ev *slack.MessageEvent, rmsg *config
 			break
 		}
 
-		if err = b.handleRateLimit(err); err != nil {
+		if err = handleRateLimit(b.Log, err); err != nil {
 			b.Log.Errorf("Could not retrieve bot information: %#v", err)
 			return err
 		}
@@ -353,7 +148,7 @@ func (b *Bslack) extractTopicOrPurpose(text string) (string, string) {
 func (b *Bslack) replaceMention(text string) string {
 	replaceFunc := func(match string) string {
 		userID := strings.Trim(match, "@<>")
-		if username := b.getUsername(userID); userID != "" {
+		if username := b.users.getUsername(userID); userID != "" {
 			return "@" + username
 		}
 		return match
@@ -397,16 +192,6 @@ func (b *Bslack) replaceCodeFence(text string) string {
 	return codeFenceRE.ReplaceAllString(text, "```")
 }
 
-func (b *Bslack) handleRateLimit(err error) error {
-	rateLimit, ok := err.(*slack.RateLimitedError)
-	if !ok {
-		return err
-	}
-	b.Log.Infof("Rate-limited by Slack. Sleeping for %v", rateLimit.RetryAfter)
-	time.Sleep(rateLimit.RetryAfter)
-	return nil
-}
-
 // getUsersInConversation returns an array of userIDs that are members of channelID
 func (b *Bslack) getUsersInConversation(channelID string) ([]string, error) {
 	channelMembers := []string{}
@@ -417,7 +202,7 @@ func (b *Bslack) getUsersInConversation(channelID string) ([]string, error) {
 
 		members, nextCursor, err := b.sc.GetUsersInConversation(queryParams)
 		if err != nil {
-			if err = b.handleRateLimit(err); err != nil {
+			if err = handleRateLimit(b.Log, err); err != nil {
 				return channelMembers, fmt.Errorf("Could not retrieve users in channels: %#v", err)
 			}
 			continue
@@ -432,3 +217,13 @@ func (b *Bslack) getUsersInConversation(channelID string) ([]string, error) {
 	}
 	return channelMembers, nil
 }
+
+func handleRateLimit(log *logrus.Entry, err error) error {
+	rateLimit, ok := err.(*slack.RateLimitedError)
+	if !ok {
+		return err
+	}
+	log.Infof("Rate-limited by Slack. Sleeping for %v", rateLimit.RetryAfter)
+	time.Sleep(rateLimit.RetryAfter)
+	return nil
+}

bridge/slack/helpers_test.go

@@ -25,7 +25,7 @@ func TestExtractTopicOrPurpose(t *testing.T) {
 
 	logger := logrus.New()
 	logger.SetOutput(ioutil.Discard)
-	cfg := &bridge.Config{Log: logger.WithFields(nil)}
+	cfg := &bridge.Config{Bridge: &bridge.Bridge{Log: logrus.NewEntry(logger)}}
 	b := newBridge(cfg)
 	for name, tc := range testcases {
 		gotChangeType, gotOutput := b.extractTopicOrPurpose(tc.input)

bridge/slack/legacy.go

@@ -55,14 +55,18 @@ func (b *BLegacy) Connect() error {
 		})
 		if b.GetString(tokenConfig) != "" {
 			b.Log.Info("Connecting using token (receiving)")
-			b.sc = slack.New(b.GetString(tokenConfig))
+			b.sc = slack.New(b.GetString(tokenConfig), slack.OptionDebug(b.GetBool("debug")))
+			b.channels = newChannelManager(b.Log, b.sc)
+			b.users = newUserManager(b.Log, b.sc)
 			b.rtm = b.sc.NewRTM()
 			go b.rtm.ManageConnection()
 			go b.handleSlack()
 		}
 	} else if b.GetString(tokenConfig) != "" {
 		b.Log.Info("Connecting using token (sending and receiving)")
-		b.sc = slack.New(b.GetString(tokenConfig))
+		b.sc = slack.New(b.GetString(tokenConfig), slack.OptionDebug(b.GetBool("debug")))
+		b.channels = newChannelManager(b.Log, b.sc)
+		b.users = newUserManager(b.Log, b.sc)
 		b.rtm = b.sc.NewRTM()
 		go b.rtm.ManageConnection()
 		go b.handleSlack()

bridge/slack/slack.go

@@ -30,20 +30,8 @@ type Bslack struct {
 	uuid         string
 	useChannelID bool
 
-	users      map[string]*slack.User
+	channels *channels
-	usersMutex sync.RWMutex
+	users    *users
-
-	channelsByID   map[string]*slack.Channel
-	channelsByName map[string]*slack.Channel
-	channelsMutex  sync.RWMutex
-
-	channelMembers      map[string][]string
-	channelMembersMutex sync.RWMutex
-
-	refreshInProgress      bool
-	earliestChannelRefresh time.Time
-	earliestUserRefresh    time.Time
-	refreshMutex           sync.Mutex
 }
 
 const (
@@ -94,14 +82,9 @@ func newBridge(cfg *bridge.Config) *Bslack {
 		cfg.Log.Fatalf("Could not create LRU cache for Slack bridge: %v", err)
 	}
 	b := &Bslack{
-		Config:                 cfg,
+		Config: cfg,
-		uuid:                   xid.New().String(),
+		uuid:   xid.New().String(),
-		cache:                  newCache,
+		cache:  newCache,
-		users:                  map[string]*slack.User{},
-		channelsByID:           map[string]*slack.Channel{},
-		channelsByName:         map[string]*slack.Channel{},
-		earliestChannelRefresh: time.Now(),
-		earliestUserRefresh:    time.Now(),
 	}
 	return b
 }
@@ -121,7 +104,12 @@ func (b *Bslack) Connect() error {
 	// If we have a token we use the Slack websocket-based RTM for both sending and receiving.
 	if token := b.GetString(tokenConfig); token != "" {
 		b.Log.Info("Connecting using token")
-		b.sc = slack.New(token)
+
+		b.sc = slack.New(token, slack.OptionDebug(b.GetBool("Debug")))
+
+		b.channels = newChannelManager(b.Log, b.sc)
+		b.users = newUserManager(b.Log, b.sc)
+
 		b.rtm = b.sc.NewRTM()
 		go b.rtm.ManageConnection()
 		go b.handleSlack()
@@ -163,9 +151,9 @@ func (b *Bslack) JoinChannel(channel config.ChannelInfo) error {
 		return nil
 	}
 
-	b.populateChannels(false)
+	b.channels.populateChannels(false)
 
-	channelInfo, err := b.getChannel(channel.Name)
+	channelInfo, err := b.channels.getChannel(channel.Name)
 	if err != nil {
 		return fmt.Errorf("could not join channel: %#v", err)
 	}
@@ -275,7 +263,7 @@ func (b *Bslack) sendRTM(msg config.Message) (string, error) {
 		return "", nil
 	}
 
-	channelInfo, err := b.getChannel(msg.Channel)
+	channelInfo, err := b.channels.getChannel(msg.Channel)
 	if err != nil {
 		return "", fmt.Errorf("could not send message: %v", err)
 	}
@@ -293,6 +281,12 @@ func (b *Bslack) sendRTM(msg config.Message) (string, error) {
 		return "", err
 	}
 
+	// Handle prefix hint for unthreaded messages.
+	if msg.ParentID == "msg-parent-not-found" {
+		msg.ParentID = ""
+		msg.Text = fmt.Sprintf("[thread]: %s", msg.Text)
+	}
+
 	// Handle message deletions.
 	if handled, err = b.deleteMessage(&msg, channelInfo); handled {
 		return msg.ID, err
@@ -345,7 +339,7 @@ func (b *Bslack) updateTopicOrPurpose(msg *config.Message, channelInfo *slack.Ch
 		if err == nil {
 			return nil
 		}
-		if err = b.handleRateLimit(err); err != nil {
+		if err = handleRateLimit(b.Log, err); err != nil {
 			return err
 		}
 	}
@@ -386,7 +380,7 @@ func (b *Bslack) deleteMessage(msg *config.Message, channelInfo *slack.Channel)
 			return true, nil
 		}
 
-		if err = b.handleRateLimit(err); err != nil {
+		if err = handleRateLimit(b.Log, err); err != nil {
 			b.Log.Errorf("Failed to delete user message from Slack: %#v", err)
 			return true, err
 		}
@@ -405,7 +399,7 @@ func (b *Bslack) editMessage(msg *config.Message, channelInfo *slack.Channel) (b
 			return true, nil
 		}
 
-		if err = b.handleRateLimit(err); err != nil {
+		if err = handleRateLimit(b.Log, err); err != nil {
 			b.Log.Errorf("Failed to edit user message on Slack: %#v", err)
 			return true, err
 		}
@@ -418,14 +412,18 @@ func (b *Bslack) postMessage(msg *config.Message, channelInfo *slack.Channel) (s
 		return "", nil
 	}
 	messageOptions := b.prepareMessageOptions(msg)
-	messageOptions = append(messageOptions, slack.MsgOptionText(msg.Text, false))
+	messageOptions = append(
+		messageOptions,
+		slack.MsgOptionText(msg.Text, false),
+		slack.MsgOptionEnableLinkUnfurl(),
+	)
 	for {
 		_, id, err := b.rtm.PostMessage(channelInfo.ID, messageOptions...)
 		if err == nil {
 			return id, nil
 		}
 
-		if err = b.handleRateLimit(err); err != nil {
+		if err = handleRateLimit(b.Log, err); err != nil {
 			b.Log.Errorf("Failed to sent user message to Slack: %#v", err)
 			return "", err
 		}

bridge/slack/users_channels.go

@@ -0,0 +1,336 @@
+package bslack
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/nlopes/slack"
+	"github.com/sirupsen/logrus"
+)
+
+const minimumRefreshInterval = 10 * time.Second
+
+type users struct {
+	log *logrus.Entry
+	sc  *slack.Client
+
+	users           map[string]*slack.User
+	usersMutex      sync.RWMutex
+	usersSyncPoints map[string]chan struct{}
+
+	refreshInProgress bool
+	earliestRefresh   time.Time
+	refreshMutex      sync.Mutex
+}
+
+func newUserManager(log *logrus.Entry, sc *slack.Client) *users {
+	return &users{
+		log:             log,
+		sc:              sc,
+		users:           make(map[string]*slack.User),
+		usersSyncPoints: make(map[string]chan struct{}),
+		earliestRefresh: time.Now(),
+	}
+}
+
+func (b *users) getUser(id string) *slack.User {
+	b.usersMutex.RLock()
+	user, ok := b.users[id]
+	b.usersMutex.RUnlock()
+	if ok {
+		return user
+	}
+	b.populateUser(id)
+	b.usersMutex.RLock()
+	defer b.usersMutex.RUnlock()
+
+	return b.users[id]
+}
+
+func (b *users) getUsername(id string) string {
+	if user := b.getUser(id); user != nil {
+		if user.Profile.DisplayName != "" {
+			return user.Profile.DisplayName
+		}
+		return user.Name
+	}
+	b.log.Warnf("Could not find user with ID '%s'", id)
+	return ""
+}
+
+func (b *users) getAvatar(id string) string {
+	if user := b.getUser(id); user != nil {
+		return user.Profile.Image48
+	}
+	return ""
+}
+
+func (b *users) populateUser(userID string) {
+	for {
+		b.usersMutex.Lock()
+		_, exists := b.users[userID]
+		if exists {
+			// already in cache
+			b.usersMutex.Unlock()
+			return
+		}
+
+		if syncPoint, ok := b.usersSyncPoints[userID]; ok {
+			// Another goroutine is already populating this user for us so wait on it to finish.
+			b.usersMutex.Unlock()
+			<-syncPoint
+			// We do not return and iterate again to check that the entry does indeed exist
+			// in case the previous query failed for some reason.
+		} else {
+			b.usersSyncPoints[userID] = make(chan struct{})
+			b.usersMutex.Unlock()
+			break
+		}
+	}
+
+	// Do not hold the lock while fetching information from Slack
+	// as this might take an unbounded amount of time.
+	b.usersMutex.Unlock()
+
+	user, err := b.sc.GetUserInfo(userID)
+	if err != nil {
+		b.log.Debugf("GetUserInfo failed for %v: %v", userID, err)
+		return
+	}
+
+	b.usersMutex.Lock()
+	defer b.usersMutex.Unlock()
+
+	// Register user information.
+	b.users[userID] = user
+
+	// Wake up any waiting goroutines and remove the synchronization point.
+	close(b.usersSyncPoints[userID])
+	delete(b.usersSyncPoints, userID)
+}
+
+func (b *users) populateUsers(wait bool) {
+	b.refreshMutex.Lock()
+	if !wait && (time.Now().Before(b.earliestRefresh) || b.refreshInProgress) {
+		b.log.Debugf("Not refreshing user list as it was done less than %v ago.", minimumRefreshInterval)
+		b.refreshMutex.Unlock()
+
+		return
+	}
+	for b.refreshInProgress {
+		b.refreshMutex.Unlock()
+		time.Sleep(time.Second)
+		b.refreshMutex.Lock()
+	}
+	b.refreshInProgress = true
+	b.refreshMutex.Unlock()
+
+	newUsers := map[string]*slack.User{}
+	pagination := b.sc.GetUsersPaginated(slack.GetUsersOptionLimit(200))
+	count := 0
+	for {
+		var err error
+		pagination, err = pagination.Next(context.Background())
+		time.Sleep(time.Second)
+		if err != nil {
+			if pagination.Done(err) {
+				break
+			}
+
+			if err = handleRateLimit(b.log, err); err != nil {
+				b.log.Errorf("Could not retrieve users: %#v", err)
+				return
+			}
+			continue
+		}
+
+		for i := range pagination.Users {
+			newUsers[pagination.Users[i].ID] = &pagination.Users[i]
+		}
+		b.log.Debugf("getting %d users", len(pagination.Users))
+		count++
+		// more > 2000 users, slack will complain and ratelimit. break
+		if count > 10 {
+			b.log.Info("Large slack detected > 2000 users, skipping loading complete userlist.")
+			break
+		}
+	}
+
+	b.usersMutex.Lock()
+	defer b.usersMutex.Unlock()
+	b.users = newUsers
+
+	b.refreshMutex.Lock()
+	defer b.refreshMutex.Unlock()
+	b.earliestRefresh = time.Now().Add(minimumRefreshInterval)
+	b.refreshInProgress = false
+}
+
+type channels struct {
+	log *logrus.Entry
+	sc  *slack.Client
+
+	channelsByID   map[string]*slack.Channel
+	channelsByName map[string]*slack.Channel
+	channelsMutex  sync.RWMutex
+
+	channelMembers      map[string][]string
+	channelMembersMutex sync.RWMutex
+
+	refreshInProgress bool
+	earliestRefresh   time.Time
+	refreshMutex      sync.Mutex
+}
+
+func newChannelManager(log *logrus.Entry, sc *slack.Client) *channels {
+	return &channels{
+		log:             log,
+		sc:              sc,
+		channelsByID:    make(map[string]*slack.Channel),
+		channelsByName:  make(map[string]*slack.Channel),
+		earliestRefresh: time.Now(),
+	}
+}
+
+func (b *channels) getChannel(channel string) (*slack.Channel, error) {
+	if strings.HasPrefix(channel, "ID:") {
+		return b.getChannelByID(strings.TrimPrefix(channel, "ID:"))
+	}
+	return b.getChannelByName(channel)
+}
+
+func (b *channels) getChannelByName(name string) (*slack.Channel, error) {
+	return b.getChannelBy(name, b.channelsByName)
+}
+
+func (b *channels) getChannelByID(id string) (*slack.Channel, error) {
+	return b.getChannelBy(id, b.channelsByID)
+}
+
+func (b *channels) getChannelBy(lookupKey string, lookupMap map[string]*slack.Channel) (*slack.Channel, error) {
+	b.channelsMutex.RLock()
+	defer b.channelsMutex.RUnlock()
+
+	if channel, ok := lookupMap[lookupKey]; ok {
+		return channel, nil
+	}
+	return nil, fmt.Errorf("channel %s not found", lookupKey)
+}
+
+func (b *channels) getChannelMembers(users *users) config.ChannelMembers {
+	b.channelMembersMutex.RLock()
+	defer b.channelMembersMutex.RUnlock()
+
+	membersInfo := config.ChannelMembers{}
+	for channelID, members := range b.channelMembers {
+		for _, member := range members {
+			channelName := ""
+			userName := ""
+			userNick := ""
+			user := users.getUser(member)
+			if user != nil {
+				userName = user.Name
+				userNick = user.Profile.DisplayName
+			}
+			channel, _ := b.getChannelByID(channelID)
+			if channel != nil {
+				channelName = channel.Name
+			}
+			memberInfo := config.ChannelMember{
+				Username:    userName,
+				Nick:        userNick,
+				UserID:      member,
+				ChannelID:   channelID,
+				ChannelName: channelName,
+			}
+			membersInfo = append(membersInfo, memberInfo)
+		}
+	}
+	return membersInfo
+}
+
+func (b *channels) registerChannel(channel slack.Channel) {
+	b.channelsMutex.Lock()
+	defer b.channelsMutex.Unlock()
+
+	b.channelsByID[channel.ID] = &channel
+	b.channelsByName[channel.Name] = &channel
+}
+
+func (b *channels) populateChannels(wait bool) {
+	b.refreshMutex.Lock()
+	if !wait && (time.Now().Before(b.earliestRefresh) || b.refreshInProgress) {
+		b.log.Debugf("Not refreshing channel list as it was done less than %v seconds ago.", minimumRefreshInterval)
+		b.refreshMutex.Unlock()
+		return
+	}
+	for b.refreshInProgress {
+		b.refreshMutex.Unlock()
+		time.Sleep(time.Second)
+		b.refreshMutex.Lock()
+	}
+	b.refreshInProgress = true
+	b.refreshMutex.Unlock()
+
+	newChannelsByID := map[string]*slack.Channel{}
+	newChannelsByName := map[string]*slack.Channel{}
+	newChannelMembers := make(map[string][]string)
+
+	// We only retrieve public and private channels, not IMs
+	// and MPIMs as those do not have a channel name.
+	queryParams := &slack.GetConversationsParameters{
+		ExcludeArchived: "true",
+		Types:           []string{"public_channel,private_channel"},
+	}
+	for {
+		channels, nextCursor, err := b.sc.GetConversations(queryParams)
+		if err != nil {
+			if err = handleRateLimit(b.log, err); err != nil {
+				b.log.Errorf("Could not retrieve channels: %#v", err)
+				return
+			}
+			continue
+		}
+
+		for i := range channels {
+			newChannelsByID[channels[i].ID] = &channels[i]
+			newChannelsByName[channels[i].Name] = &channels[i]
+			// also find all the members in every channel
+			// comment for now, issues on big slacks
+			/*
+				members, err := b.getUsersInConversation(channels[i].ID)
+				if err != nil {
+					if err = b.handleRateLimit(err); err != nil {
+						b.Log.Errorf("Could not retrieve channel members: %#v", err)
+						return
+					}
+					continue
+				}
+				newChannelMembers[channels[i].ID] = members
+			*/
+		}
+
+		if nextCursor == "" {
+			break
+		}
+		queryParams.Cursor = nextCursor
+	}
+
+	b.channelsMutex.Lock()
+	defer b.channelsMutex.Unlock()
+	b.channelsByID = newChannelsByID
+	b.channelsByName = newChannelsByName
+
+	b.channelMembersMutex.Lock()
+	defer b.channelMembersMutex.Unlock()
+	b.channelMembers = newChannelMembers
+
+	b.refreshMutex.Lock()
+	defer b.refreshMutex.Unlock()
+	b.earliestRefresh = time.Now().Add(minimumRefreshInterval)
+	b.refreshInProgress = false
+}

bridge/sshchat/sshchat.go

@@ -9,7 +9,6 @@ import (
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/42wim/matterbridge/bridge/helper"
 	"github.com/shazow/ssh-chat/sshd"
-	"github.com/sirupsen/logrus"
 )
 
 type Bsshchat struct {
@@ -134,7 +133,7 @@ func (b *Bsshchat) handleSSHChat() error {
 			res := strings.Split(stripPrompt(b.r.Text()), ":")
 			if res[0] == "-> Set theme" {
 				wait = false
-				logrus.Debugf("mono found, allowing")
+				b.Log.Debugf("mono found, allowing")
 				continue
 			}
 			if !wait {

bridge/telegram/handlers.go

@@ -144,6 +144,9 @@ func (b *Btelegram) handleRecv(updates <-chan tgbotapi.Update) {
 		// quote the previous message
 		b.handleQuoting(&rmsg, message)
 
+		// handle entities (adding URLs)
+		b.handleEntities(&rmsg, message)
+
 		if rmsg.Text != "" || len(rmsg.Extra) > 0 {
 			rmsg.Text = helper.RemoveEmptyNewLines(rmsg.Text)
 			// channels don't have (always?) user information. see #410
@@ -245,6 +248,15 @@ func (b *Btelegram) handleDownload(rmsg *config.Message, message *tgbotapi.Messa
 	if err != nil {
 		return err
 	}
+	if strings.HasSuffix(name, ".webp") && b.GetBool("MediaConvertWebPToPNG") {
+		b.Log.Debugf("WebP to PNG conversion enabled, converting %s", name)
+		err := helper.ConvertWebPToPNG(data)
+		if err != nil {
+			b.Log.Errorf("conversion failed: %s", err)
+		} else {
+			name = strings.Replace(name, ".webp", ".png", 1)
+		}
+	}
 	helper.HandleDownloadData(b.Log, rmsg, name, message.Caption, "", data, b.General)
 	return nil
 }
@@ -344,3 +356,22 @@ func (b *Btelegram) handleQuote(message, quoteNick, quoteMessage string) string
 	format = strings.Replace(format, "{QUOTEMESSAGE}", quoteMessage, -1)
 	return format
 }
+
+// handleEntities handles messageEntities
+func (b *Btelegram) handleEntities(rmsg *config.Message, message *tgbotapi.Message) {
+	if message.Entities == nil {
+		return
+	}
+	// for now only do URL replacements
+	for _, e := range *message.Entities {
+		if e.Type == "text_link" {
+			url, err := e.ParseURL()
+			if err != nil {
+				b.Log.Errorf("entity text_link url parse failed: %s", err)
+				continue
+			}
+			link := rmsg.Text[e.Offset : e.Offset+e.Length]
+			rmsg.Text = strings.Replace(rmsg.Text, link, url.String(), 1)
+		}
+	}
+}

bridge/telegram/html.go

@@ -3,7 +3,6 @@ package btelegram
 import (
 	"bytes"
 	"html"
-	"io"
 
 	"github.com/russross/blackfriday"
 )
@@ -33,7 +32,7 @@ func (options *customHTML) Header(out *bytes.Buffer, text func() bool, level int
 	options.Paragraph(out, text)
 }
 
-func (options *customHTML) HRule(out io.ByteWriter) {
+func (options *customHTML) HRule(out *bytes.Buffer) {
 	out.WriteByte('\n') //nolint:errcheck
 }
 
@@ -54,16 +53,13 @@ func (options *customHTML) ListItem(out *bytes.Buffer, text []byte, flags int) {
 }
 
 func makeHTML(input string) string {
-	extensions := blackfriday.NoIntraEmphasis |
+	return string(blackfriday.Markdown([]byte(input),
-		blackfriday.FencedCode |
+		&customHTML{blackfriday.HtmlRenderer(blackfriday.HTML_USE_XHTML|blackfriday.HTML_SKIP_IMAGES, "", "")},
-		blackfriday.Autolink |
+		blackfriday.EXTENSION_NO_INTRA_EMPHASIS|
-		blackfriday.SpaceHeadings |
+			blackfriday.EXTENSION_FENCED_CODE|
-		blackfriday.HeadingIDs |
+			blackfriday.EXTENSION_AUTOLINK|
-		blackfriday.BackslashLineBreak |
+			blackfriday.EXTENSION_SPACE_HEADERS|
-		blackfriday.DefinitionLists
+			blackfriday.EXTENSION_HEADER_IDS|
-
+			blackfriday.EXTENSION_BACKSLASH_LINE_BREAK|
-	renderer := &customHTML{blackfriday.NewHTMLRenderer(blackfriday.HTMLRendererParameters{
+			blackfriday.EXTENSION_DEFINITION_LISTS))
-		Flags: blackfriday.UseXHTML | blackfriday.SkipImages,
-	})}
-	return string(blackfriday.Run([]byte(input), blackfriday.WithExtensions(extensions), blackfriday.WithRenderer(renderer)))
 }

bridge/whatsapp/handlers.go

@@ -0,0 +1,104 @@
+package bwhatsapp
+
+import (
+	"strings"
+	"time"
+
+	"github.com/42wim/matterbridge/bridge/config"
+
+	"github.com/matterbridge/go-whatsapp"
+
+	whatsappExt "github.com/matterbridge/mautrix-whatsapp/whatsapp-ext"
+)
+
+/*
+Implement handling messages coming from WhatsApp
+Check:
+- https://github.com/Rhymen/go-whatsapp#add-message-handlers
+- https://github.com/Rhymen/go-whatsapp/blob/master/handler.go
+- https://github.com/tulir/mautrix-whatsapp/tree/master/whatsapp-ext for more advanced command handling
+*/
+
+// HandleError received from WhatsApp
+func (b *Bwhatsapp) HandleError(err error) {
+	b.Log.Errorf("%v", err) // TODO implement proper handling? at least respond to different error types
+}
+
+// HandleTextMessage sent from WhatsApp, relay it to the brige
+func (b *Bwhatsapp) HandleTextMessage(message whatsapp.TextMessage) {
+	if message.Info.FromMe { // || !strings.Contains(strings.ToLower(message.Text), "@echo") {
+		return
+	}
+	// whatsapp sends last messages to show context , cut them
+	if message.Info.Timestamp < b.startedAt {
+		return
+	}
+
+	messageTime := time.Unix(int64(message.Info.Timestamp), 0) // TODO check how behaves between timezones
+	groupJid := message.Info.RemoteJid
+
+	senderJid := message.Info.SenderJid
+	if len(senderJid) == 0 {
+		// TODO workaround till https://github.com/Rhymen/go-whatsapp/issues/86 resolved
+		senderJid = *message.Info.Source.Participant
+	}
+
+	// translate sender's Jid to the nicest username we can get
+	senderName := b.getSenderName(senderJid)
+	if senderName == "" {
+		senderName = "Someone" // don't expose telephone number
+	}
+
+	extText := message.Info.Source.Message.ExtendedTextMessage
+	if extText != nil && extText.ContextInfo != nil && extText.ContextInfo.MentionedJid != nil {
+		// handle user mentions
+		for _, mentionedJid := range extText.ContextInfo.MentionedJid {
+			numberAndSuffix := strings.SplitN(mentionedJid, "@", 2)
+
+			// mentions comes as telephone numbers and we don't want to expose it to other bridges
+			// replace it with something more meaninful to others
+			mention := b.getSenderNotify(numberAndSuffix[0] + whatsappExt.NewUserSuffix)
+			if mention == "" {
+				mention = "someone"
+			}
+			message.Text = strings.Replace(message.Text, "@"+numberAndSuffix[0], "@"+mention, 1)
+		}
+	}
+
+	b.Log.Debugf("<= Sending message from %s on %s to gateway", senderJid, b.Account)
+	rmsg := config.Message{
+		UserID:    senderJid,
+		Username:  senderName,
+		Text:      message.Text,
+		Timestamp: messageTime,
+		Channel:   groupJid,
+		Account:   b.Account,
+		Protocol:  b.Protocol,
+		Extra:     make(map[string][]interface{}),
+		//		ParentID: TODO, // TODO handle thread replies  // map from Info.QuotedMessageID string
+		//	Event     string    `json:"event"`
+		//	Gateway   string  // will be added during message processing
+		ID: message.Info.Id}
+
+	if avatarURL, exists := b.userAvatars[senderJid]; exists {
+		rmsg.Avatar = avatarURL
+	}
+
+	b.Log.Debugf("<= Message is %#v", rmsg)
+	b.Remote <- rmsg
+}
+
+//
+//func (b *Bwhatsapp) HandleImageMessage(message whatsapp.ImageMessage) {
+//	fmt.Println(message) // TODO implement
+//}
+//
+//func (b *Bwhatsapp) HandleVideoMessage(message whatsapp.VideoMessage) {
+//	fmt.Println(message) // TODO implement
+//}
+//
+//func (b *Bwhatsapp) HandleJsonMessage(message string) {
+//	fmt.Println(message) // TODO implement
+//}
+// TODO HandleRawMessage
+// TODO HandleAudioMessage

bridge/whatsapp/helpers.go

@@ -0,0 +1,84 @@
+package bwhatsapp
+
+import (
+	"encoding/gob"
+	"errors"
+	"os"
+
+	qrcodeTerminal "github.com/Baozisoftware/qrcode-terminal-go"
+	"github.com/matterbridge/go-whatsapp"
+)
+
+func qrFromTerminal(invert bool) chan string {
+	qr := make(chan string)
+	go func() {
+		terminal := qrcodeTerminal.New()
+		if invert {
+			terminal = qrcodeTerminal.New2(qrcodeTerminal.ConsoleColors.BrightWhite, qrcodeTerminal.ConsoleColors.BrightBlack, qrcodeTerminal.QRCodeRecoveryLevels.Medium)
+		}
+
+		terminal.Get(<-qr).Print()
+	}()
+
+	return qr
+}
+
+func (b *Bwhatsapp) readSession() (whatsapp.Session, error) {
+	session := whatsapp.Session{}
+	sessionFile := b.Config.GetString(sessionFile)
+
+	if sessionFile == "" {
+		return session, errors.New("if you won't set SessionFile then you will need to scan QR code on every restart")
+	}
+
+	file, err := os.Open(sessionFile)
+	if err != nil {
+		return session, err
+	}
+	defer file.Close()
+	decoder := gob.NewDecoder(file)
+	err = decoder.Decode(&session)
+	if err != nil {
+		return session, err
+	}
+	return session, nil
+}
+
+func (b *Bwhatsapp) writeSession(session whatsapp.Session) error {
+	sessionFile := b.Config.GetString(sessionFile)
+
+	if sessionFile == "" {
+		// we already sent a warning while starting the bridge, so let's be quiet here
+		return nil
+	}
+
+	file, err := os.Create(sessionFile)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+	encoder := gob.NewEncoder(file)
+	err = encoder.Encode(session)
+
+	return err
+}
+
+func (b *Bwhatsapp) getSenderName(senderJid string) string {
+	if sender, exists := b.users[senderJid]; exists {
+		if sender.Name != "" {
+			return sender.Name
+		}
+		// if user is not in phone contacts
+		// it is the most obvious scenario unless you sync your phone contacts with some remote updated source
+		// users can change it in their WhatsApp settings -> profile -> click on Avatar
+		return sender.Notify
+	}
+	return ""
+}
+
+func (b *Bwhatsapp) getSenderNotify(senderJid string) string {
+	if sender, exists := b.users[senderJid]; exists {
+		return sender.Notify
+	}
+	return ""
+}

bridge/whatsapp/whatsapp.go

@@ -0,0 +1,305 @@
+package bwhatsapp
+
+import (
+	"crypto/rand"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/42wim/matterbridge/bridge"
+	"github.com/42wim/matterbridge/bridge/config"
+
+	"github.com/matterbridge/go-whatsapp"
+
+	whatsappExt "github.com/matterbridge/mautrix-whatsapp/whatsapp-ext"
+)
+
+const (
+	// Account config parameters
+	cfgNumber         = "Number"
+	qrOnWhiteTerminal = "QrOnWhiteTerminal"
+	sessionFile       = "SessionFile"
+)
+
+// Bwhatsapp Bridge structure keeping all the information needed for relying
+type Bwhatsapp struct {
+	*bridge.Config
+
+	// https://github.com/Rhymen/go-whatsapp/blob/c31092027237441cffba1b9cb148eadf7c83c3d2/session.go#L18-L21
+	session *whatsapp.Session
+	conn    *whatsapp.Conn
+	// https://github.com/tulir/mautrix-whatsapp/blob/master/whatsapp-ext/whatsapp.go
+	connExt   *whatsappExt.ExtendedConn
+	startedAt uint64
+
+	users       map[string]whatsapp.Contact
+	userAvatars map[string]string
+}
+
+// New Create a new WhatsApp bridge. This will be called for each [whatsapp.<server>] entry you have in the config file
+func New(cfg *bridge.Config) bridge.Bridger {
+	number := cfg.GetString(cfgNumber)
+	if number == "" {
+		cfg.Log.Fatalf("Missing configuration for WhatsApp bridge: Number")
+	}
+
+	b := &Bwhatsapp{
+		Config: cfg,
+
+		users:       make(map[string]whatsapp.Contact),
+		userAvatars: make(map[string]string),
+	}
+	return b
+}
+
+// Connect to WhatsApp. Required implementation of the Bridger interface
+// https://github.com/42wim/matterbridge/blob/2cfd880cdb0df29771bf8f31df8d990ab897889d/bridge/bridge.go#L11-L16
+func (b *Bwhatsapp) Connect() error {
+	b.RLock() // TODO do we need locking for Whatsapp?
+	defer b.RUnlock()
+
+	number := b.GetString(cfgNumber)
+	if number == "" {
+		return errors.New("WhatsApp's telephone Number need to be configured")
+	}
+
+	// https://github.com/Rhymen/go-whatsapp#creating-a-connection
+	b.Log.Debugln("Connecting to WhatsApp..")
+	conn, err := whatsapp.NewConn(20 * time.Second)
+	if err != nil {
+		return errors.New("failed to connect to WhatsApp: " + err.Error())
+	}
+
+	b.conn = conn
+	b.connExt = whatsappExt.ExtendConn(b.conn)
+	// TODO do we want to use it? b.connExt.SetClientName("Matterbridge WhatsApp bridge", "mb-wa")
+
+	b.conn.AddHandler(b)
+	b.Log.Debugln("WhatsApp connection successful")
+
+	// load existing session in order to keep it between restarts
+	if b.session == nil {
+		var session whatsapp.Session
+		session, err = b.readSession()
+
+		if err == nil {
+			b.Log.Debugln("Restoring WhatsApp session..")
+
+			// https://github.com/Rhymen/go-whatsapp#restore
+			session, err = b.conn.RestoreSession(session)
+			if err != nil {
+				// TODO return or continue to normal login?
+				// restore session connection timed out (I couldn't get over it without logging in again)
+				return errors.New("failed to restore session: " + err.Error())
+			}
+
+			b.session = &session
+			b.Log.Debugln("Session restored successfully!")
+		} else {
+			b.Log.Warn(err.Error())
+		}
+	}
+
+	// login to a new session
+	if b.session == nil {
+		err = b.Login()
+		if err != nil {
+			return err
+		}
+	}
+	b.startedAt = uint64(time.Now().Unix())
+
+	_, err = b.conn.Contacts()
+	if err != nil {
+		return fmt.Errorf("error on update of contacts: %v", err)
+	}
+
+	// map all the users
+	for id, contact := range b.conn.Store.Contacts {
+		if !isGroupJid(id) && id != "status@broadcast" {
+			// it is user
+			b.users[id] = contact
+		}
+	}
+
+	// get user avatar asynchronously
+	go func() {
+		b.Log.Debug("Getting user avatars..")
+
+		for jid := range b.users {
+			info, err := b.connExt.GetProfilePicThumb(jid)
+			if err != nil {
+				b.Log.Warnf("Could not get profile photo of %s: %v", jid, err)
+
+			} else {
+				// TODO any race conditions here?
+				b.userAvatars[jid] = info.URL
+			}
+		}
+		b.Log.Debug("Finished getting avatars..")
+	}()
+
+	return nil
+}
+
+// Login to WhatsApp creating a new session. This will require to scan a QR code on your mobile device
+func (b *Bwhatsapp) Login() error {
+	b.Log.Debugln("Logging in..")
+
+	invert := b.GetBool(qrOnWhiteTerminal) // false is the default
+	qrChan := qrFromTerminal(invert)
+
+	session, err := b.conn.Login(qrChan)
+	if err != nil {
+		b.Log.Warnln("Failed to log in:", err)
+		return err
+	}
+	b.session = &session
+
+	b.Log.Infof("Logged into session: %#v", session)
+	b.Log.Infof("Connection: %#v", b.conn)
+
+	err = b.writeSession(session)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "error saving session: %v\n", err)
+	}
+
+	// TODO change connection strings to configured ones longClientName:"github.com/rhymen/go-whatsapp", shortClientName:"go-whatsapp"}" prefix=whatsapp
+	// TODO get also a nice logo
+
+	// TODO notification about unplugged and dead battery
+	// conn.Info: Wid, Pushname, Connected, Battery, Plugged
+
+	return nil
+}
+
+// Disconnect is called while reconnecting to the bridge
+// TODO 42wim Documentation would be helpful on when reconnects happen and what should be done in this function
+// Required implementation of the Bridger interface
+// https://github.com/42wim/matterbridge/blob/2cfd880cdb0df29771bf8f31df8d990ab897889d/bridge/bridge.go#L11-L16
+func (b *Bwhatsapp) Disconnect() error {
+	// We could Logout, but that would close the session completely and would require a new QR code scan
+	// https://github.com/Rhymen/go-whatsapp/blob/c31092027237441cffba1b9cb148eadf7c83c3d2/session.go#L377-L381
+	return nil
+}
+
+func isGroupJid(identifier string) bool {
+	return strings.HasSuffix(identifier, "@g.us") || strings.HasSuffix(identifier, "@temp")
+}
+
+// JoinChannel Join a WhatsApp group specified in gateway config as channel='number-id@g.us' or channel='Channel name'
+// Required implementation of the Bridger interface
+// https://github.com/42wim/matterbridge/blob/2cfd880cdb0df29771bf8f31df8d990ab897889d/bridge/bridge.go#L11-L16
+func (b *Bwhatsapp) JoinChannel(channel config.ChannelInfo) error {
+	byJid := isGroupJid(channel.Name)
+
+	// verify if we are member of the given group
+	if byJid {
+		// channel.Name specifies static group jID, not the name
+		if _, exists := b.conn.Store.Contacts[channel.Name]; !exists {
+			return fmt.Errorf("account doesn't belong to group with jid %s", channel.Name)
+		}
+	} else {
+		// channel.Name specifies group name that might change, warn about it
+		var jids []string
+		for id, contact := range b.conn.Store.Contacts {
+			if isGroupJid(id) && contact.Name == channel.Name {
+				jids = append(jids, id)
+			}
+		}
+
+		switch len(jids) {
+		case 0:
+			// didn't match any group - print out possibilites
+			// TODO sort
+			// copy b;
+			//sort.Slice(people, func(i, j int) bool {
+			//	return people[i].Age > people[j].Age
+			//})
+			for id, contact := range b.conn.Store.Contacts {
+				if isGroupJid(id) {
+					b.Log.Infof("%s %s", contact.Jid, contact.Name)
+				}
+			}
+			return fmt.Errorf("please specify group's JID from the list above instead of the name '%s'", channel.Name)
+
+		case 1:
+			return fmt.Errorf("group name might change. Please configure gateway with channel=\"%v\" instead of channel=\"%v\"", jids[0], channel.Name)
+
+		default:
+			return fmt.Errorf("there is more than one group with name '%s'. Please specify one of JIDs as channel name: %v", channel.Name, jids)
+		}
+	}
+
+	return nil
+}
+
+// Send a message from the bridge to WhatsApp
+// Required implementation of the Bridger interface
+// https://github.com/42wim/matterbridge/blob/2cfd880cdb0df29771bf8f31df8d990ab897889d/bridge/bridge.go#L11-L16
+func (b *Bwhatsapp) Send(msg config.Message) (string, error) {
+	b.Log.Debugf("=> Receiving %#v", msg)
+
+	// Delete message
+	if msg.Event == config.EventMsgDelete {
+		if msg.ID == "" {
+			// No message ID in case action is executed on a message sent before the bridge was started
+			// and then the bridge cache doesn't have this message ID mapped
+
+			// TODO 42wim Doesn't the app get clogged with a ton of IDs after some time of running?
+			// WhatsApp allows to set any ID so in that case we could use external IDs and don't do mapping
+			// but external IDs are not set
+			return "", nil
+		}
+		// TODO delete message on WhatsApp https://github.com/Rhymen/go-whatsapp/issues/100
+		return "", nil
+	}
+
+	// Edit message
+	if msg.ID != "" {
+		b.Log.Debugf("updating message with id %s", msg.ID)
+
+		msg.Text += " (edited)"
+		// TODO handle edit as a message reply with updated text
+	}
+
+	//// TODO Handle Upload a file
+	//if msg.Extra != nil {
+	//	for _, rmsg := range helper.HandleExtra(&msg, b.General) {
+	//		b.c.SendMessage(roomID, rmsg.Username+rmsg.Text)
+	//	}
+	//	if len(msg.Extra["file"]) > 0 {
+	//		return b.handleUploadFile(&msg, roomID)
+	//	}
+	//}
+
+	// Post text message
+	text := whatsapp.TextMessage{
+		Info: whatsapp.MessageInfo{
+			RemoteJid: msg.Channel, // which equals to group id
+		},
+		Text: msg.Username + msg.Text,
+	}
+
+	b.Log.Debugf("=> Sending %#v", msg)
+
+	// create message ID
+	// TODO follow and act if https://github.com/Rhymen/go-whatsapp/issues/101 implemented
+	bytes := make([]byte, 10)
+	if _, err := rand.Read(bytes); err != nil {
+		b.Log.Warn(err.Error())
+	}
+	text.Info.Id = strings.ToUpper(hex.EncodeToString(bytes))
+
+	err := b.conn.Send(text)
+
+	return text.Info.Id, err
+}
+
+// TODO do we want that? to allow login with QR code from a bridged channel? https://github.com/tulir/mautrix-whatsapp/blob/513eb18e2d59bada0dd515ee1abaaf38a3bfe3d5/commands.go#L76
+//func (b *Bwhatsapp) Command(cmd string) string {
+//	return ""
+//}

bridge/xmpp/xmpp.go

@@ -17,6 +17,7 @@ type Bxmpp struct {
 	xc      *xmpp.Client
 	xmppMap map[string]string
 	*bridge.Config
+	startTime time.Time
 }
 
 func New(cfg *bridge.Config) bridge.Bridger {
@@ -153,6 +154,7 @@ func (b *Bxmpp) xmppKeepAlive() chan bool {
 func (b *Bxmpp) handleXMPP() error {
 	var ok bool
 	var msgid string
+	b.startTime = time.Now()
 	done := b.xmppKeepAlive()
 	defer close(done)
 	for {
@@ -164,15 +166,27 @@ func (b *Bxmpp) handleXMPP() error {
 		case xmpp.Chat:
 			if v.Type == "groupchat" {
 				b.Log.Debugf("== Receiving %#v", v)
+				event := ""
 				// skip invalid messages
 				if b.skipMessage(v) {
 					continue
 				}
+				if strings.Contains(v.Text, "has set the subject to:") {
+					event = config.EventTopicChange
+				}
 				msgid = v.ID
 				if v.ReplaceID != "" {
 					msgid = v.ReplaceID
 				}
-				rmsg := config.Message{Username: b.parseNick(v.Remote), Text: v.Text, Channel: b.parseChannel(v.Remote), Account: b.Account, UserID: v.Remote, ID: msgid}
+				rmsg := config.Message{
+					Username: b.parseNick(v.Remote),
+					Text:     v.Text,
+					Channel:  b.parseChannel(v.Remote),
+					Account:  b.Account,
+					UserID:   v.Remote,
+					ID:       msgid,
+					Event:    event,
+				}
 
 				// check if we have an action event
 				rmsg.Text, ok = b.replaceAction(rmsg.Text)
@@ -259,6 +273,11 @@ func (b *Bxmpp) skipMessage(message xmpp.Chat) bool {
 		return true
 	}
 
+	// do not show subjects on connect #732
+	if strings.Contains(message.Text, "has set the subject to:") && time.Since(b.startTime) < time.Second*5 {
+		return true
+	}
+
 	// skip delayed messages
 	t := time.Time{}
 	return message.Stamp != t

bridge/zulip/zulip.go

@@ -4,6 +4,8 @@ import (
 	"encoding/json"
 	"io/ioutil"
 	"strconv"
+	"strings"
+	"sync"
 	"time"
 
 	"github.com/42wim/matterbridge/bridge"
@@ -17,6 +19,7 @@ type Bzulip struct {
 	bot     *gzb.Bot
 	streams map[int]string
 	*bridge.Config
+	sync.RWMutex
 }
 
 func New(cfg *bridge.Config) bridge.Bridger {
@@ -100,14 +103,46 @@ func (b *Bzulip) getChannel(id int) string {
 
 func (b *Bzulip) handleQueue() error {
 	for {
-		messages, _ := b.q.GetEvents()
+		messages, err := b.q.GetEvents()
+		switch err {
+		case gzb.BackoffError:
+			time.Sleep(time.Second * 5)
+		case gzb.NoJSONError:
+			b.Log.Error("Response wasn't JSON, server down or restarting? sleeping 10 seconds")
+			time.Sleep(time.Second * 10)
+		case gzb.BadEventQueueError:
+			b.Log.Info("got a bad event queue id error, reconnecting")
+			b.bot.Queues = nil
+			for {
+				b.q, err = b.bot.RegisterAll()
+				if err != nil {
+					b.Log.Errorf("reconnecting failed: %s. Sleeping 10 seconds", err)
+					time.Sleep(time.Second * 10)
+				}
+				break
+			}
+		case gzb.HeartbeatError:
+			b.Log.Debug("heartbeat received.")
+		default:
+			b.Log.Debugf("receiving error: %#v", err)
+		}
+		if err != nil {
+			continue
+		}
 		for _, m := range messages {
 			b.Log.Debugf("== Receiving %#v", m)
 			// ignore our own messages
 			if m.SenderEmail == b.GetString("login") {
 				continue
 			}
-			rmsg := config.Message{Username: m.SenderFullName, Text: m.Content, Channel: b.getChannel(m.StreamID), Account: b.Account, UserID: strconv.Itoa(m.SenderID), Avatar: m.AvatarURL}
+			rmsg := config.Message{
+				Username: m.SenderFullName,
+				Text:     m.Content,
+				Channel:  b.getChannel(m.StreamID) + "/topic:" + m.Subject,
+				Account:  b.Account,
+				UserID:   strconv.Itoa(m.SenderID),
+				Avatar:   m.AvatarURL,
+			}
 			b.Log.Debugf("<= Sending message from %s on %s to gateway", rmsg.Username, b.Account)
 			b.Log.Debugf("<= Message is %#v", rmsg)
 			b.Remote <- rmsg
@@ -118,9 +153,11 @@ func (b *Bzulip) handleQueue() error {
 }
 
 func (b *Bzulip) sendMessage(msg config.Message) (string, error) {
-	topic := "matterbridge"
+	topic := ""
-	if b.GetString("topic") != "" {
+	if strings.Contains(msg.Channel, "/topic:") {
-		topic = b.GetString("topic")
+		res := strings.Split(msg.Channel, "/topic:")
+		topic = res[1]
+		msg.Channel = res[0]
 	}
 	m := gzb.Message{
 		Stream:  msg.Channel,

changelog.md

@@ -1,3 +1,46 @@
+# v1.14.0
+
+## Breaking
+* zulip: Need to specify /topic:mytopic for channel configuration (zulip). (#751)
+
+## New features
+* whatsapp: new protocol added. Add initial WhatsApp support (#711) Thanks to @KrzysztofMadejski
+* facebook messenger: new protocol via matterbridge api. See https://github.com/VictorNine/fbridge/ for more information.
+* general: Add scripting (tengo) support for every incoming message (#731). See `TengoModifyMessage`
+* general: Allow regexs in ignoreNicks. Closes #690 (#720)
+* general: Support rewriting messages from relaybots using ExtractNicks. Fixes #466 (#730). See `ExtractNicks` in matterbridge.toml.sample
+* general: refactor Make all loggers derive from non-default instance (#728). Thanks to @Helcaraxan
+* rocketchat: add support for the rocketchat API. Sending to rocketchat now supports uploading of files, editing and deleting of messages.
+* discord: Support join/leaves from discord. Closes #654 (#721)
+* discord: Allow sending discriminator with Discord username (#726). See `UseDiscriminator` in matterbridge.toml.sample
+* slack: Add extra debug option (slack). See `Debug` in the slack section in matterbridge.toml.sample
+* telegram: Add support for URL in messageEntities (telegram). Fixes #735 (#736)
+* telegram: Add MediaConvertWebPToPNG option (telegram). (#741). See `MediaConvertWebPToPNG` in matterbridge.toml.sample
+
+## Enhancements
+* general: Fail gracefully on incorrect human input. Fixes #739 (#740)
+* matrix: Detect html nicks in RemoteNickFormat (matrix). Fixes #696 (#719)
+* matrix: Send notices on join/parts (matrix). Fixes #712 (#716)
+
+## Bugfix
+* general: Handle file upload/download only once for each message (#742)
+* zulip: Fix error handling on bad event queue id (zulip). Closes #694
+* zulip: Keep reconnecting until succeed (zulip) (#737)
+* irc: add support for (older) unrealircd versions. #708
+* irc: Support quits from irc correctly. Fixes #722 (#724)
+* matrix: Send username when uploading video/images (matrix). Fixes #715 (#717)
+* matrix: Trim <p> and </p> tags (matrix). Closes #686 (#753)
+* slack: Hint at thread replies when messages are unthreaded (slack) (#684)
+* slack: Fix race-condition in populateUser() (#767)
+* xmpp: Do not send topic changes on connect (xmpp). Fixes #732 (#733)
+* telegram: Fix regression in HTML handling (telegram). Closes #734
+* discord: Do not relay any bot messages (discord) (#743)
+* rocketchat: Do not send duplicate messages (rocketchat). Fixes #745 (#752)
+
+## Contributors
+This release couldn't exist without the following contributors:
+@Helcaraxan, @KrzysztofMadejski, @AJolly, @DeclanHoare
+
 # v1.13.1
 
 This release fixes go modules issues because of https://github.com/labstack/echo/issues/1272

contrib/example.tengo

@@ -0,0 +1,2 @@
+text := import("text")
+msgText=text.re_replace("matterbridge",msgText,"matterbridge (https://github.com/42wim/matterbridge)")

gateway/bench.tengo

@@ -0,0 +1,5 @@
+text := import("text")
+if text.re_match("blah",msgText) {
+    msgText="replaced by this"
+    msgUsername="fakeuser"
+}

gateway/bridgemap/bridgemap.go

@@ -13,6 +13,7 @@ import (
 	"github.com/42wim/matterbridge/bridge/sshchat"
 	"github.com/42wim/matterbridge/bridge/steam"
 	"github.com/42wim/matterbridge/bridge/telegram"
+	"github.com/42wim/matterbridge/bridge/whatsapp"
 	"github.com/42wim/matterbridge/bridge/xmpp"
 	"github.com/42wim/matterbridge/bridge/zulip"
 )
@@ -30,6 +31,7 @@ var FullMap = map[string]bridge.Factory{
 	"sshchat":      bsshchat.New,
 	"steam":        bsteam.New,
 	"telegram":     btelegram.New,
+	"whatsapp":     bwhatsapp.New,
 	"xmpp":         bxmpp.New,
 	"zulip":        bzulip.New,
 }

gateway/gateway.go

@@ -1,6 +1,7 @@
 package gateway
 
 import (
+	"io/ioutil"
 	"os"
 	"regexp"
 	"strings"
@@ -8,7 +9,8 @@ import (
 
 	"github.com/42wim/matterbridge/bridge"
 	"github.com/42wim/matterbridge/bridge/config"
-	"github.com/hashicorp/golang-lru"
+	"github.com/d5/tengo/script"
+	lru "github.com/hashicorp/golang-lru"
 	"github.com/peterhellberg/emojilib"
 	"github.com/sirupsen/logrus"
 )
@@ -24,6 +26,8 @@ type Gateway struct {
 	Message        chan config.Message
 	Name           string
 	Messages       *lru.Cache
+
+	logger *logrus.Entry
 }
 
 type BrMsgID struct {
@@ -32,25 +36,30 @@ type BrMsgID struct {
 	ChannelID string
 }
 
-var flog *logrus.Entry
+const apiProtocol = "api"
 
-const (
+// New creates a new Gateway object associated with the specified router and
-	apiProtocol = "api"
+// following the given configuration.
-)
+func New(rootLogger *logrus.Logger, cfg *config.Gateway, r *Router) *Gateway {
+	logger := rootLogger.WithFields(logrus.Fields{"prefix": "gateway"})
 
-func New(cfg config.Gateway, r *Router) *Gateway {
-	flog = logrus.WithFields(logrus.Fields{"prefix": "gateway"})
-	gw := &Gateway{Channels: make(map[string]*config.ChannelInfo), Message: r.Message,
-		Router: r, Bridges: make(map[string]*bridge.Bridge), Config: r.Config}
 	cache, _ := lru.New(5000)
-	gw.Messages = cache
+	gw := &Gateway{
-	if err := gw.AddConfig(&cfg); err != nil {
+		Channels: make(map[string]*config.ChannelInfo),
-		flog.Errorf("AddConfig failed: %s", err)
+		Message:  r.Message,
+		Router:   r,
+		Bridges:  make(map[string]*bridge.Bridge),
+		Config:   r.Config,
+		Messages: cache,
+		logger:   logger,
+	}
+	if err := gw.AddConfig(cfg); err != nil {
+		logger.Errorf("Failed to add configuration to gateway: %#v", err)
 	}
 	return gw
 }
 
-// Find the canonical ID that the message is keyed under in cache
+// FindCanonicalMsgID returns the ID under which a message was stored in the cache.
 func (gw *Gateway) FindCanonicalMsgID(protocol string, mID string) string {
 	ID := protocol + " " + mID
 	if gw.Messages.Contains(ID) {
@@ -70,16 +79,22 @@ func (gw *Gateway) FindCanonicalMsgID(protocol string, mID string) string {
 	return ""
 }
 
+// AddBridge sets up a new bridge in the gateway object with the specified configuration.
 func (gw *Gateway) AddBridge(cfg *config.Bridge) error {
 	br := gw.Router.getBridge(cfg.Account)
 	if br == nil {
 		br = bridge.New(cfg)
 		br.Config = gw.Router.Config
 		br.General = &gw.BridgeValues().General
-		// set logging
+		br.Log = gw.logger.WithFields(logrus.Fields{"prefix": br.Protocol})
-		br.Log = logrus.WithFields(logrus.Fields{"prefix": "bridge"})
+		brconfig := &bridge.Config{
-		brconfig := &bridge.Config{Remote: gw.Message, Log: logrus.WithFields(logrus.Fields{"prefix": br.Protocol}), Bridge: br}
+			Remote: gw.Message,
+			Bridge: br,
+		}
 		// add the actual bridger for this protocol to this bridge using the bridgeMap
+		if _, ok := gw.Router.BridgeMap[br.Protocol]; !ok {
+			gw.logger.Fatalf("Incorrect protocol %s specified in gateway configuration %s, exiting.", br.Protocol, cfg.Account)
+		}
 		br.Bridger = gw.Router.BridgeMap[br.Protocol](brconfig)
 	}
 	gw.mapChannelsToBridge(br)
@@ -87,11 +102,12 @@ func (gw *Gateway) AddBridge(cfg *config.Bridge) error {
 	return nil
 }
 
+// AddConfig associates a new configuration with the gateway object.
 func (gw *Gateway) AddConfig(cfg *config.Gateway) error {
 	gw.Name = cfg.Name
 	gw.MyConfig = cfg
 	if err := gw.mapChannels(); err != nil {
-		flog.Errorf("mapChannels() failed: %s", err)
+		gw.logger.Errorf("mapChannels() failed: %s", err)
 	}
 	for _, br := range append(gw.MyConfig.In, append(gw.MyConfig.InOut, gw.MyConfig.Out...)...) {
 		br := br //scopelint
@@ -113,20 +129,20 @@ func (gw *Gateway) mapChannelsToBridge(br *bridge.Bridge) {
 
 func (gw *Gateway) reconnectBridge(br *bridge.Bridge) {
 	if err := br.Disconnect(); err != nil {
-		flog.Errorf("Disconnect() %s failed: %s", br.Account, err)
+		gw.logger.Errorf("Disconnect() %s failed: %s", br.Account, err)
 	}
 	time.Sleep(time.Second * 5)
 RECONNECT:
-	flog.Infof("Reconnecting %s", br.Account)
+	gw.logger.Infof("Reconnecting %s", br.Account)
 	err := br.Connect()
 	if err != nil {
-		flog.Errorf("Reconnection failed: %s. Trying again in 60 seconds", err)
+		gw.logger.Errorf("Reconnection failed: %s. Trying again in 60 seconds", err)
 		time.Sleep(time.Second * 60)
 		goto RECONNECT
 	}
 	br.Joined = make(map[string]bool)
 	if err := br.JoinChannels(); err != nil {
-		flog.Errorf("JoinChannels() %s failed: %s", br.Account, err)
+		gw.logger.Errorf("JoinChannels() %s failed: %s", br.Account, err)
 	}
 }
 
@@ -140,13 +156,23 @@ func (gw *Gateway) mapChannelConfig(cfg []config.Bridge, direction string) {
 			br.Channel = strings.ToLower(br.Channel)
 		}
 		if strings.HasPrefix(br.Account, "mattermost.") && strings.HasPrefix(br.Channel, "#") {
-			flog.Errorf("Mattermost channels do not start with a #: remove the # in %s", br.Channel)
+			gw.logger.Errorf("Mattermost channels do not start with a #: remove the # in %s", br.Channel)
+			os.Exit(1)
+		}
+		if strings.HasPrefix(br.Account, "zulip.") && !strings.Contains(br.Channel, "/topic:") {
+			gw.logger.Errorf("Breaking change, since matterbridge 1.14.0 zulip channels need to specify the topic with channel/topic:mytopic in %s of %s", br.Channel, br.Account)
 			os.Exit(1)
 		}
 		ID := br.Channel + br.Account
 		if _, ok := gw.Channels[ID]; !ok {
-			channel := &config.ChannelInfo{Name: br.Channel, Direction: direction, ID: ID, Options: br.Options, Account: br.Account,
+			channel := &config.ChannelInfo{
-				SameChannel: make(map[string]bool)}
+				Name:        br.Channel,
+				Direction:   direction,
+				ID:          ID,
+				Options:     br.Options,
+				Account:     br.Account,
+				SameChannel: make(map[string]bool),
+			}
 			channel.SameChannel[gw.Name] = br.SameChannel
 			gw.Channels[channel.ID] = channel
 		} else {
@@ -174,10 +200,38 @@ func (gw *Gateway) getDestChannel(msg *config.Message, dest bridge.Bridge) []con
 		return channels
 	}
 
+	// discord join/leave is for the whole bridge, isn't a per channel join/leave
+	if msg.Event == config.EventJoinLeave && getProtocol(msg) == "discord" && msg.Channel == "" {
+		for _, channel := range gw.Channels {
+			if channel.Account == dest.Account && strings.Contains(channel.Direction, "out") &&
+				gw.validGatewayDest(msg) {
+				channels = append(channels, *channel)
+			}
+		}
+		return channels
+	}
+
+	// irc quit is for the whole bridge, isn't a per channel quit.
+	// channel is empty when we quit
+	if msg.Event == config.EventJoinLeave && getProtocol(msg) == "irc" && msg.Channel == "" {
+		// if we only have one channel on this irc bridge it's got to be the sending one.
+		// don't send it back
+		if dest.Account == msg.Account && len(dest.Channels) == 1 && dest.Protocol == "irc" {
+			return channels
+		}
+		for _, channel := range gw.Channels {
+			if channel.Account == dest.Account && strings.Contains(channel.Direction, "out") &&
+				gw.validGatewayDest(msg) {
+				channels = append(channels, *channel)
+			}
+		}
+		return channels
+	}
+
 	// if source channel is in only, do nothing
 	for _, channel := range gw.Channels {
 		// lookup the channel from the message
-		if channel.ID == getChannelID(*msg) {
+		if channel.ID == getChannelID(msg) {
 			// we only have destinations if the original message is from an "in" (sending) channel
 			if !strings.Contains(channel.Direction, "in") {
 				return channels
@@ -186,11 +240,11 @@ func (gw *Gateway) getDestChannel(msg *config.Message, dest bridge.Bridge) []con
 		}
 	}
 	for _, channel := range gw.Channels {
-		if _, ok := gw.Channels[getChannelID(*msg)]; !ok {
+		if _, ok := gw.Channels[getChannelID(msg)]; !ok {
 			continue
 		}
 
-		// do samechannelgateway flogic
+		// do samechannelgateway logic
 		if channel.SameChannel[msg.Gateway] {
 			if msg.Channel == channel.Name && msg.Account != dest.Account {
 				channels = append(channels, *channel)
@@ -204,7 +258,7 @@ func (gw *Gateway) getDestChannel(msg *config.Message, dest bridge.Bridge) []con
 	return channels
 }
 
-func (gw *Gateway) getDestMsgID(msgID string, dest *bridge.Bridge, channel config.ChannelInfo) string {
+func (gw *Gateway) getDestMsgID(msgID string, dest *bridge.Bridge, channel *config.ChannelInfo) string {
 	if res, ok := gw.Messages.Get(msgID); ok {
 		IDs := res.([]*BrMsgID)
 		for _, id := range IDs {
@@ -233,42 +287,10 @@ func (gw *Gateway) ignoreTextEmpty(msg *config.Message) bool {
 			len(msg.Extra[config.EventFileFailureSize]) > 0) {
 		return false
 	}
-	flog.Debugf("ignoring empty message %#v from %s", msg, msg.Account)
+	gw.logger.Debugf("ignoring empty message %#v from %s", msg, msg.Account)
 	return true
 }
 
-// ignoreTexts returns true if msg.Text matches any of the input regexes.
-func (gw *Gateway) ignoreTexts(msg *config.Message, input []string) bool {
-	for _, entry := range input {
-		if entry == "" {
-			continue
-		}
-		// TODO do not compile regexps everytime
-		re, err := regexp.Compile(entry)
-		if err != nil {
-			flog.Errorf("incorrect regexp %s for %s", entry, msg.Account)
-			continue
-		}
-		if re.MatchString(msg.Text) {
-			flog.Debugf("matching %s. ignoring %s from %s", entry, msg.Text, msg.Account)
-			return true
-		}
-	}
-	return false
-}
-
-// ignoreNicks returns true if msg.Username matches any of the input regexes.
-func (gw *Gateway) ignoreNicks(msg *config.Message, input []string) bool {
-	// is the username in IgnoreNicks field
-	for _, entry := range input {
-		if msg.Username == entry {
-			flog.Debugf("ignoring %s from %s", msg.Username, msg.Account)
-			return true
-		}
-	}
-	return false
-}
-
 func (gw *Gateway) ignoreMessage(msg *config.Message) bool {
 	// if we don't have the bridge, ignore it
 	if _, ok := gw.Bridges[msg.Account]; !ok {
@@ -277,14 +299,14 @@ func (gw *Gateway) ignoreMessage(msg *config.Message) bool {
 
 	igNicks := strings.Fields(gw.Bridges[msg.Account].GetString("IgnoreNicks"))
 	igMessages := strings.Fields(gw.Bridges[msg.Account].GetString("IgnoreMessages"))
-	if gw.ignoreTextEmpty(msg) || gw.ignoreNicks(msg, igNicks) || gw.ignoreTexts(msg, igMessages) {
+	if gw.ignoreTextEmpty(msg) || gw.ignoreText(msg.Username, igNicks) || gw.ignoreText(msg.Text, igMessages) {
 		return true
 	}
 
 	return false
 }
 
-func (gw *Gateway) modifyUsername(msg config.Message, dest *bridge.Bridge) string {
+func (gw *Gateway) modifyUsername(msg *config.Message, dest *bridge.Bridge) string {
 	br := gw.Bridges[msg.Account]
 	msg.Protocol = br.Protocol
 	if dest.GetBool("StripNick") {
@@ -300,7 +322,7 @@ func (gw *Gateway) modifyUsername(msg config.Message, dest *bridge.Bridge) strin
 		// TODO move compile to bridge init somewhere
 		re, err := regexp.Compile(search)
 		if err != nil {
-			flog.Errorf("regexp in %s failed: %s", msg.Account, err)
+			gw.logger.Errorf("regexp in %s failed: %s", msg.Account, err)
 			break
 		}
 		msg.Username = re.ReplaceAllString(msg.Username, replace)
@@ -328,7 +350,7 @@ func (gw *Gateway) modifyUsername(msg config.Message, dest *bridge.Bridge) strin
 	return nick
 }
 
-func (gw *Gateway) modifyAvatar(msg config.Message, dest *bridge.Bridge) string {
+func (gw *Gateway) modifyAvatar(msg *config.Message, dest *bridge.Bridge) string {
 	iconurl := dest.GetString("IconURL")
 	iconurl = strings.Replace(iconurl, "{NICK}", msg.Username, -1)
 	if msg.Avatar == "" {
@@ -338,6 +360,10 @@ func (gw *Gateway) modifyAvatar(msg config.Message, dest *bridge.Bridge) string
 }
 
 func (gw *Gateway) modifyMessage(msg *config.Message) {
+	if err := modifyMessageTengo(gw.BridgeValues().General.TengoModifyMessage, msg); err != nil {
+		gw.logger.Errorf("TengoModifyMessage failed: %s", err)
+	}
+
 	// replace :emoji: to unicode
 	msg.Text = emojilib.Replace(msg.Text)
 
@@ -349,55 +375,68 @@ func (gw *Gateway) modifyMessage(msg *config.Message) {
 		// TODO move compile to bridge init somewhere
 		re, err := regexp.Compile(search)
 		if err != nil {
-			flog.Errorf("regexp in %s failed: %s", msg.Account, err)
+			gw.logger.Errorf("regexp in %s failed: %s", msg.Account, err)
 			break
 		}
 		msg.Text = re.ReplaceAllString(msg.Text, replace)
 	}
 
+	gw.handleExtractNicks(msg)
+
 	// messages from api have Gateway specified, don't overwrite
 	if msg.Protocol != apiProtocol {
 		msg.Gateway = gw.Name
 	}
 }
 
-// SendMessage sends a message (with specified parentID) to the channel on the selected destination bridge.
+// SendMessage sends a message (with specified parentID) to the channel on the selected
-// returns a message id and error.
+// destination bridge and returns a message ID or an error.
-func (gw *Gateway) SendMessage(origmsg config.Message, dest *bridge.Bridge, channel config.ChannelInfo, canonicalParentMsgID string) (string, error) {
+func (gw *Gateway) SendMessage(
-	msg := origmsg
+	rmsg *config.Message,
+	dest *bridge.Bridge,
+	channel *config.ChannelInfo,
+	canonicalParentMsgID string,
+) (string, error) {
+	msg := *rmsg
 	// Only send the avatar download event to ourselves.
 	if msg.Event == config.EventAvatarDownload {
-		if channel.ID != getChannelID(origmsg) {
+		if channel.ID != getChannelID(rmsg) {
 			return "", nil
 		}
 	} else {
 		// do not send to ourself for any other event
-		if channel.ID == getChannelID(origmsg) {
+		if channel.ID == getChannelID(rmsg) {
 			return "", nil
 		}
 	}
 
 	// Too noisy to log like other events
 	if msg.Event != config.EventUserTyping {
-		flog.Debugf("=> Sending %#v from %s (%s) to %s (%s)", msg, msg.Account, origmsg.Channel, dest.Account, channel.Name)
+		gw.logger.Debugf("=> Sending %#v from %s (%s) to %s (%s)", msg, msg.Account, rmsg.Channel, dest.Account, channel.Name)
 	}
 
 	msg.Channel = channel.Name
-	msg.Avatar = gw.modifyAvatar(origmsg, dest)
+	msg.Avatar = gw.modifyAvatar(rmsg, dest)
-	msg.Username = gw.modifyUsername(origmsg, dest)
+	msg.Username = gw.modifyUsername(rmsg, dest)
 
-	msg.ID = gw.getDestMsgID(origmsg.Protocol+" "+origmsg.ID, dest, channel)
+	msg.ID = gw.getDestMsgID(rmsg.Protocol+" "+rmsg.ID, dest, channel)
 
 	// for api we need originchannel as channel
 	if dest.Protocol == apiProtocol {
-		msg.Channel = origmsg.Channel
+		msg.Channel = rmsg.Channel
 	}
 
-	msg.ParentID = gw.getDestMsgID(origmsg.Protocol+" "+canonicalParentMsgID, dest, channel)
+	msg.ParentID = gw.getDestMsgID(rmsg.Protocol+" "+canonicalParentMsgID, dest, channel)
 	if msg.ParentID == "" {
 		msg.ParentID = canonicalParentMsgID
 	}
 
+	// if the parentID is still empty and we have a parentID set in the original message
+	// this means that we didn't find it in the cache so set it "msg-parent-not-found"
+	if msg.ParentID == "" && rmsg.ParentID != "" {
+		msg.ParentID = "msg-parent-not-found"
+	}
+
 	// if we are using mattermost plugin account, send messages to MattermostPlugin channel
 	// that can be picked up by the mattermost matterbridge plugin
 	if dest.Account == "mattermost.plugin" {
@@ -411,7 +450,7 @@ func (gw *Gateway) SendMessage(origmsg config.Message, dest *bridge.Bridge, chan
 
 	// append the message ID (mID) from this bridge (dest) to our brMsgIDs slice
 	if mID != "" {
-		flog.Debugf("mID %s: %s", dest.Account, mID)
+		gw.logger.Debugf("mID %s: %s", dest.Account, mID)
 		return mID, nil
 		//brMsgIDs = append(brMsgIDs, &BrMsgID{dest, dest.Protocol + " " + mID, channel.ID})
 	}
@@ -422,10 +461,60 @@ func (gw *Gateway) validGatewayDest(msg *config.Message) bool {
 	return msg.Gateway == gw.Name
 }
 
-func getChannelID(msg config.Message) string {
+func getChannelID(msg *config.Message) string {
 	return msg.Channel + msg.Account
 }
 
 func isAPI(account string) bool {
 	return strings.HasPrefix(account, "api.")
 }
+
+// ignoreText returns true if text matches any of the input regexes.
+func (gw *Gateway) ignoreText(text string, input []string) bool {
+	for _, entry := range input {
+		if entry == "" {
+			continue
+		}
+		// TODO do not compile regexps everytime
+		re, err := regexp.Compile(entry)
+		if err != nil {
+			gw.logger.Errorf("incorrect regexp %s", entry)
+			continue
+		}
+		if re.MatchString(text) {
+			gw.logger.Debugf("matching %s. ignoring %s", entry, text)
+			return true
+		}
+	}
+	return false
+}
+
+func getProtocol(msg *config.Message) string {
+	p := strings.Split(msg.Account, ".")
+	return p[0]
+}
+
+func modifyMessageTengo(filename string, msg *config.Message) error {
+	if filename == "" {
+		return nil
+	}
+	res, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return err
+	}
+	s := script.New(res)
+	_ = s.Add("msgText", msg.Text)
+	_ = s.Add("msgUsername", msg.Username)
+	_ = s.Add("msgAccount", msg.Account)
+	_ = s.Add("msgChannel", msg.Channel)
+	c, err := s.Compile()
+	if err != nil {
+		return err
+	}
+	if err := c.Run(); err != nil {
+		return err
+	}
+	msg.Text = c.Get("msgText").String()
+	msg.Username = c.Get("msgUsername").String()
+	return nil
+}

gateway/gateway_test.go

@@ -2,12 +2,15 @@ package gateway
 
 import (
 	"fmt"
+	"io/ioutil"
 	"strconv"
 	"testing"
 
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/42wim/matterbridge/gateway/bridgemap"
+	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/suite"
 )
 
 var testconfig = []byte(`
@@ -159,8 +162,10 @@ const (
 )
 
 func maketestRouter(input []byte) *Router {
-	cfg := config.NewConfigFromString(input)
+	logger := logrus.New()
-	r, err := NewRouter(cfg, bridgemap.FullMap)
+	logger.SetOutput(ioutil.Discard)
+	cfg := config.NewConfigFromString(logger, input)
+	r, err := NewRouter(logger, cfg, bridgemap.FullMap)
 	if err != nil {
 		fmt.Println(err)
 	}
@@ -387,7 +392,23 @@ func TestGetDestChannelAdvanced(t *testing.T) {
 	assert.Equal(t, map[string]int{"bridge3": 4, "bridge": 9, "announcements": 3, "bridge2": 4}, hits)
 }
 
-func TestIgnoreTextEmpty(t *testing.T) {
+type ignoreTestSuite struct {
+	suite.Suite
+
+	gw *Gateway
+}
+
+func TestIgnoreSuite(t *testing.T) {
+	s := &ignoreTestSuite{}
+	suite.Run(t, s)
+}
+
+func (s *ignoreTestSuite) SetupSuite() {
+	logger := logrus.New()
+	logger.SetOutput(ioutil.Discard)
+	s.gw = &Gateway{logger: logrus.NewEntry(logger)}
+}
+func (s *ignoreTestSuite) TestIgnoreTextEmpty() {
 	extraFile := make(map[string][]interface{})
 	extraAttach := make(map[string][]interface{})
 	extraFailure := make(map[string][]interface{})
@@ -424,78 +445,85 @@ func TestIgnoreTextEmpty(t *testing.T) {
 			output: true,
 		},
 	}
-	gw := &Gateway{}
 	for testname, testcase := range msgTests {
-		output := gw.ignoreTextEmpty(testcase.input)
+		output := s.gw.ignoreTextEmpty(testcase.input)
-		assert.Equalf(t, testcase.output, output, "case '%s' failed", testname)
+		s.Assert().Equalf(testcase.output, output, "case '%s' failed", testname)
 	}
 
 }
 
-func TestIgnoreTexts(t *testing.T) {
+func (s *ignoreTestSuite) TestIgnoreTexts() {
 	msgTests := map[string]struct {
-		input  *config.Message
+		input  string
 		re     []string
 		output bool
 	}{
 		"no regex": {
-			input:  &config.Message{Text: "a text message"},
+			input:  "a text message",
 			re:     []string{},
 			output: false,
 		},
 		"simple regex": {
-			input:  &config.Message{Text: "a text message"},
+			input:  "a text message",
 			re:     []string{"text"},
 			output: true,
 		},
 		"multiple regex fail": {
-			input:  &config.Message{Text: "a text message"},
+			input:  "a text message",
 			re:     []string{"abc", "123$"},
 			output: false,
 		},
 		"multiple regex pass": {
-			input:  &config.Message{Text: "a text message"},
+			input:  "a text message",
 			re:     []string{"lala", "sage$"},
 			output: true,
 		},
 	}
-	gw := &Gateway{}
 	for testname, testcase := range msgTests {
-		output := gw.ignoreTexts(testcase.input, testcase.re)
+		output := s.gw.ignoreText(testcase.input, testcase.re)
-		assert.Equalf(t, testcase.output, output, "case '%s' failed", testname)
+		s.Assert().Equalf(testcase.output, output, "case '%s' failed", testname)
 	}
 }
 
-func TestIgnoreNicks(t *testing.T) {
+func (s *ignoreTestSuite) TestIgnoreNicks() {
 	msgTests := map[string]struct {
-		input  *config.Message
+		input  string
 		re     []string
 		output bool
 	}{
 		"no entry": {
-			input:  &config.Message{Username: "user", Text: "a text message"},
+			input:  "user",
 			re:     []string{},
 			output: false,
 		},
 		"one entry": {
-			input:  &config.Message{Username: "user", Text: "a text message"},
+			input:  "user",
 			re:     []string{"user"},
 			output: true,
 		},
 		"multiple entries": {
-			input:  &config.Message{Username: "user", Text: "a text message"},
+			input:  "user",
 			re:     []string{"abc", "user"},
 			output: true,
 		},
 		"multiple entries fail": {
-			input:  &config.Message{Username: "user", Text: "a text message"},
+			input:  "user",
 			re:     []string{"abc", "def"},
 			output: false,
 		},
 	}
-	gw := &Gateway{}
 	for testname, testcase := range msgTests {
-		output := gw.ignoreNicks(testcase.input, testcase.re)
+		output := s.gw.ignoreText(testcase.input, testcase.re)
-		assert.Equalf(t, testcase.output, output, "case '%s' failed", testname)
+		s.Assert().Equalf(testcase.output, output, "case '%s' failed", testname)
+	}
+}
+
+func BenchmarkTengo(b *testing.B) {
+	msg := &config.Message{Username: "user", Text: "blah testing", Account: "protocol.account", Channel: "mychannel"}
+	for n := 0; n < b.N; n++ {
+		err := modifyMessageTengo("bench.tengo", msg)
+		if err != nil {
+			return
+		}
 	}
 }

gateway/handlers.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"strings"
 	"time"
 
 	"github.com/42wim/matterbridge/bridge"
@@ -39,7 +40,7 @@ func (r *Router) handleEventGetChannelMembers(msg *config.Message) {
 		for _, br := range gw.Bridges {
 			if msg.Account == br.Account {
 				cMembers := msg.Extra[config.EventGetChannelMembers][0].(config.ChannelMembers)
-				flog.Debugf("Syncing channelmembers from %s", msg.Account)
+				r.logger.Debugf("Syncing channelmembers from %s", msg.Account)
 				br.SetChannelMembers(&cMembers)
 				return
 			}
@@ -57,7 +58,7 @@ func (r *Router) handleEventRejoinChannels(msg *config.Message) {
 			if msg.Account == br.Account {
 				br.Joined = make(map[string]bool)
 				if err := br.JoinChannels(); err != nil {
-					flog.Errorf("channel join failed for %s: %s", msg.Account, err)
+					r.logger.Errorf("channel join failed for %s: %s", msg.Account, err)
 				}
 			}
 		}
@@ -93,13 +94,13 @@ func (gw *Gateway) handleFiles(msg *config.Message) {
 		if gw.BridgeValues().General.MediaServerUpload != "" {
 			// Use MediaServerUpload. Upload using a PUT HTTP request and basicauth.
 			if err := gw.handleFilesUpload(&fi); err != nil {
-				flog.Error(err)
+				gw.logger.Error(err)
 				continue
 			}
 		} else {
 			// Use MediaServerPath. Place the file on the current filesystem.
 			if err := gw.handleFilesLocal(&fi); err != nil {
-				flog.Error(err)
+				gw.logger.Error(err)
 				continue
 			}
 		}
@@ -107,7 +108,7 @@ func (gw *Gateway) handleFiles(msg *config.Message) {
 		// Download URL.
 		durl := gw.BridgeValues().General.MediaServerDownload + "/" + sha1sum + "/" + fi.Name
 
-		flog.Debugf("mediaserver download URL = %s", durl)
+		gw.logger.Debugf("mediaserver download URL = %s", durl)
 
 		// We uploaded/placed the file successfully. Add the SHA and URL.
 		extra := msg.Extra["file"][i].(config.FileInfo)
@@ -132,7 +133,7 @@ func (gw *Gateway) handleFilesUpload(fi *config.FileInfo) error {
 		return fmt.Errorf("mediaserver upload failed, could not create request: %#v", err)
 	}
 
-	flog.Debugf("mediaserver upload url: %s", url)
+	gw.logger.Debugf("mediaserver upload url: %s", url)
 
 	req.Header.Set("Content-Type", "binary/octet-stream")
 	_, err = client.Do(req)
@@ -153,7 +154,7 @@ func (gw *Gateway) handleFilesLocal(fi *config.FileInfo) error {
 	}
 
 	path := dir + "/" + fi.Name
-	flog.Debugf("mediaserver path placing file: %s", path)
+	gw.logger.Debugf("mediaserver path placing file: %s", path)
 
 	err = ioutil.WriteFile(path, *fi.Data, os.ModePerm)
 	if err != nil {
@@ -186,36 +187,36 @@ func (gw *Gateway) ignoreEvent(event string, dest *bridge.Bridge) bool {
 
 // handleMessage makes sure the message get sent to the correct bridge/channels.
 // Returns an array of msg ID's
-func (gw *Gateway) handleMessage(msg config.Message, dest *bridge.Bridge) []*BrMsgID {
+func (gw *Gateway) handleMessage(rmsg *config.Message, dest *bridge.Bridge) []*BrMsgID {
 	var brMsgIDs []*BrMsgID
 
 	// if we have an attached file, or other info
-	if msg.Extra != nil && len(msg.Extra[config.EventFileFailureSize]) != 0 && msg.Text == "" {
+	if rmsg.Extra != nil && len(rmsg.Extra[config.EventFileFailureSize]) != 0 && rmsg.Text == "" {
 		return brMsgIDs
 	}
 
-	if gw.ignoreEvent(msg.Event, dest) {
+	if gw.ignoreEvent(rmsg.Event, dest) {
 		return brMsgIDs
 	}
 
 	// broadcast to every out channel (irc QUIT)
-	if msg.Channel == "" && msg.Event != config.EventJoinLeave {
+	if rmsg.Channel == "" && rmsg.Event != config.EventJoinLeave {
-		flog.Debug("empty channel")
+		gw.logger.Debug("empty channel")
 		return brMsgIDs
 	}
 
 	// Get the ID of the parent message in thread
 	var canonicalParentMsgID string
-	if msg.ParentID != "" && dest.GetBool("PreserveThreading") {
+	if rmsg.ParentID != "" && dest.GetBool("PreserveThreading") {
-		canonicalParentMsgID = gw.FindCanonicalMsgID(msg.Protocol, msg.ParentID)
+		canonicalParentMsgID = gw.FindCanonicalMsgID(rmsg.Protocol, rmsg.ParentID)
 	}
 
-	origmsg := msg
+	channels := gw.getDestChannel(rmsg, *dest)
-	channels := gw.getDestChannel(&msg, *dest)
+	for idx := range channels {
-	for _, channel := range channels {
+		channel := &channels[idx]
-		msgID, err := gw.SendMessage(origmsg, dest, channel, canonicalParentMsgID)
+		msgID, err := gw.SendMessage(rmsg, dest, channel, canonicalParentMsgID)
 		if err != nil {
-			flog.Errorf("SendMessage failed: %s", err)
+			gw.logger.Errorf("SendMessage failed: %s", err)
 			continue
 		}
 		if msgID == "" {
@@ -225,3 +226,41 @@ func (gw *Gateway) handleMessage(msg config.Message, dest *bridge.Bridge) []*BrM
 	}
 	return brMsgIDs
 }
+
+func (gw *Gateway) handleExtractNicks(msg *config.Message) {
+	var err error
+	br := gw.Bridges[msg.Account]
+	for _, outer := range br.GetStringSlice2D("ExtractNicks") {
+		search := outer[0]
+		replace := outer[1]
+		msg.Username, msg.Text, err = extractNick(search, replace, msg.Username, msg.Text)
+		if err != nil {
+			gw.logger.Errorf("regexp in %s failed: %s", msg.Account, err)
+			break
+		}
+	}
+}
+
+// extractNick searches for a username (based on "search" a regular expression).
+// if this matches it extracts a nick (based on "extract" another regular expression) from text
+// and replaces username with this result.
+// returns error if the regexp doesn't compile.
+func extractNick(search, extract, username, text string) (string, string, error) {
+	re, err := regexp.Compile(search)
+	if err != nil {
+		return username, text, err
+	}
+	if re.MatchString(username) {
+		re, err = regexp.Compile(extract)
+		if err != nil {
+			return username, text, err
+		}
+		res := re.FindAllStringSubmatch(text, 1)
+		// only replace if we have exactly 1 match
+		if len(res) > 0 && len(res[0]) == 2 {
+			username = res[0][1]
+			text = strings.Replace(text, res[0][0], "", 1)
+		}
+	}
+	return username, text, nil
+}

gateway/handlers_test.go

@@ -0,0 +1,75 @@
+package gateway
+
+import (
+	"github.com/42wim/matterbridge/bridge"
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/stretchr/testify/assert"
+
+	"testing"
+)
+
+func TestIgnoreEvent(t *testing.T) {
+	eventTests := map[string]struct {
+		input  string
+		dest   *bridge.Bridge
+		output bool
+	}{
+		"avatar mattermost": {
+			input:  config.EventAvatarDownload,
+			dest:   &bridge.Bridge{Protocol: "mattermost"},
+			output: false,
+		},
+		"avatar slack": {
+			input:  config.EventAvatarDownload,
+			dest:   &bridge.Bridge{Protocol: "slack"},
+			output: true,
+		},
+		"avatar telegram": {
+			input:  config.EventAvatarDownload,
+			dest:   &bridge.Bridge{Protocol: "telegram"},
+			output: false,
+		},
+	}
+	gw := &Gateway{}
+	for testname, testcase := range eventTests {
+		output := gw.ignoreEvent(testcase.input, testcase.dest)
+		assert.Equalf(t, testcase.output, output, "case '%s' failed", testname)
+	}
+
+}
+
+func TestExtractNick(t *testing.T) {
+	eventTests := map[string]struct {
+		search         string
+		extract        string
+		username       string
+		text           string
+		resultUsername string
+		resultText     string
+	}{
+		"test1": {
+			search:         "fromgitter",
+			extract:        "<(.*?)>\\s+",
+			username:       "fromgitter",
+			text:           "<userx> blahblah",
+			resultUsername: "userx",
+			resultText:     "blahblah",
+		},
+		"test2": {
+			search: "<.*?bot>",
+			//extract:        `\((.*?)\)\s+`,
+			extract:        "\\((.*?)\\)\\s+",
+			username:       "<matterbot>",
+			text:           "(userx) blahblah (abc) test",
+			resultUsername: "userx",
+			resultText:     "blahblah (abc) test",
+		},
+	}
+	//	gw := &Gateway{}
+	for testname, testcase := range eventTests {
+		resultUsername, resultText, _ := extractNick(testcase.search, testcase.extract, testcase.username, testcase.text)
+		assert.Equalf(t, testcase.resultUsername, resultUsername, "case '%s' failed", testname)
+		assert.Equalf(t, testcase.resultText, resultText, "case '%s' failed", testname)
+	}
+
+}

gateway/router.go

@@ -7,31 +7,40 @@ import (
 
 	"github.com/42wim/matterbridge/bridge"
 	"github.com/42wim/matterbridge/bridge/config"
-	samechannelgateway "github.com/42wim/matterbridge/gateway/samechannel"
+	"github.com/42wim/matterbridge/gateway/samechannel"
+	"github.com/sirupsen/logrus"
 )
 
 type Router struct {
 	config.Config
+	sync.RWMutex
 
 	BridgeMap        map[string]bridge.Factory
 	Gateways         map[string]*Gateway
 	Message          chan config.Message
 	MattermostPlugin chan config.Message
-	sync.RWMutex
+
+	logger *logrus.Entry
 }
 
-func NewRouter(cfg config.Config, bridgeMap map[string]bridge.Factory) (*Router, error) {
+// NewRouter initializes a new Matterbridge router for the specified configuration and
+// sets up all required gateways.
+func NewRouter(rootLogger *logrus.Logger, cfg config.Config, bridgeMap map[string]bridge.Factory) (*Router, error) {
+	logger := rootLogger.WithFields(logrus.Fields{"prefix": "router"})
+
 	r := &Router{
 		Config:           cfg,
 		BridgeMap:        bridgeMap,
 		Message:          make(chan config.Message),
 		MattermostPlugin: make(chan config.Message),
 		Gateways:         make(map[string]*Gateway),
+		logger:           logger,
 	}
-	sgw := samechannelgateway.New(cfg)
+	sgw := samechannel.New(cfg)
-	gwconfigs := sgw.GetConfig()
+	gwconfigs := append(sgw.GetConfig(), cfg.BridgeValues().Gateway...)
 
-	for _, entry := range append(gwconfigs, cfg.BridgeValues().Gateway...) {
+	for idx := range gwconfigs {
+		entry := &gwconfigs[idx]
 		if !entry.Enable {
 			continue
 		}
@@ -41,21 +50,23 @@ func NewRouter(cfg config.Config, bridgeMap map[string]bridge.Factory) (*Router,
 		if _, ok := r.Gateways[entry.Name]; ok {
 			return nil, fmt.Errorf("Gateway with name %s already exists", entry.Name)
 		}
-		r.Gateways[entry.Name] = New(entry, r)
+		r.Gateways[entry.Name] = New(rootLogger, entry, r)
 	}
 	return r, nil
 }
 
+// Start will connect all gateways belonging to this router and subsequently route messages
+// between them.
 func (r *Router) Start() error {
 	m := make(map[string]*bridge.Bridge)
 	for _, gw := range r.Gateways {
-		flog.Infof("Parsing gateway %s", gw.Name)
+		r.logger.Infof("Parsing gateway %s", gw.Name)
 		for _, br := range gw.Bridges {
 			m[br.Account] = br
 		}
 	}
 	for _, br := range m {
-		flog.Infof("Starting bridge: %s ", br.Account)
+		r.logger.Infof("Starting bridge: %s ", br.Account)
 		err := br.Connect()
 		if err != nil {
 			e := fmt.Errorf("Bridge %s failed to start: %v", br.Account, err)
@@ -77,13 +88,13 @@ func (r *Router) Start() error {
 	for _, gw := range r.Gateways {
 		for i, br := range gw.Bridges {
 			if br.Bridger == nil {
-				flog.Errorf("removing failed bridge %s", i)
+				r.logger.Errorf("removing failed bridge %s", i)
 				delete(gw.Bridges, i)
 			}
 		}
 	}
 	go r.handleReceive()
-	go r.updateChannelMembers()
+	//go r.updateChannelMembers()
 	return nil
 }
 
@@ -91,7 +102,7 @@ func (r *Router) Start() error {
 // otherwise returns false
 func (r *Router) disableBridge(br *bridge.Bridge, err error) bool {
 	if r.BridgeValues().General.IgnoreFailureOnStart {
-		flog.Error(err)
+		r.logger.Error(err)
 		// setting this bridge empty
 		*br = bridge.Bridge{}
 		return true
@@ -114,6 +125,7 @@ func (r *Router) handleReceive() {
 		r.handleEventGetChannelMembers(&msg)
 		r.handleEventFailure(&msg)
 		r.handleEventRejoinChannels(&msg)
+		idx := 0
 		for _, gw := range r.Gateways {
 			// record all the message ID's of the different bridges
 			var msgIDs []*BrMsgID
@@ -122,14 +134,17 @@ func (r *Router) handleReceive() {
 			}
 			msg.Timestamp = time.Now()
 			gw.modifyMessage(&msg)
-			gw.handleFiles(&msg)
+			if idx == 0 {
+				gw.handleFiles(&msg)
+			}
 			for _, br := range gw.Bridges {
-				msgIDs = append(msgIDs, gw.handleMessage(msg, br)...)
+				msgIDs = append(msgIDs, gw.handleMessage(&msg, br)...)
 			}
 			// only add the message ID if it doesn't already exists
 			if _, ok := gw.Messages.Get(msg.Protocol + " " + msg.ID); !ok && msg.ID != "" {
 				gw.Messages.Add(msg.Protocol+" "+msg.ID, msgIDs)
 			}
+			idx++
 		}
 	}
 }
@@ -146,9 +161,9 @@ func (r *Router) updateChannelMembers() {
 				if br.Protocol != "slack" {
 					continue
 				}
-				flog.Debugf("sending %s to %s", config.EventGetChannelMembers, br.Account)
+				r.logger.Debugf("sending %s to %s", config.EventGetChannelMembers, br.Account)
 				if _, err := br.Send(config.Message{Event: config.EventGetChannelMembers}); err != nil {
-					flog.Errorf("updateChannelMembers: %s", err)
+					r.logger.Errorf("updateChannelMembers: %s", err)
 				}
 			}
 		}

gateway/samechannel/samechannel.go

@@ -1,4 +1,4 @@
-package samechannelgateway
+package samechannel
 
 import (
 	"github.com/42wim/matterbridge/bridge/config"

gateway/samechannel/samechannel_test.go

@@ -1,9 +1,11 @@
-package samechannelgateway
+package samechannel
 
 import (
+	"io/ioutil"
 	"testing"
 
 	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -66,7 +68,9 @@ var (
 )
 
 func TestGetConfig(t *testing.T) {
-	cfg := config.NewConfigFromString([]byte(testConfig))
+	logger := logrus.New()
+	logger.SetOutput(ioutil.Discard)
+	cfg := config.NewConfigFromString(logger, []byte(testConfig))
 	sgw := New(cfg)
 	configs := sgw.GetConfig()
 	assert.Equal(t, []config.Gateway{expectedConfig}, configs)

go.mod

@@ -2,14 +2,17 @@ module github.com/42wim/matterbridge
 
 require (
 	github.com/42wim/go-gitter v0.0.0-20170828205020-017310c2d557
+	github.com/Baozisoftware/qrcode-terminal-go v0.0.0-20170407111555-c0650d8dff0f
 	github.com/BurntSushi/toml v0.0.0-20170318202913-d94612f9fc14 // indirect
+	github.com/Jeffail/gabs v1.1.1 // indirect
 	github.com/Philipp15b/go-steam v1.0.1-0.20180818081528-681bd9573329
 	github.com/bwmarrin/discordgo v0.19.0
+	github.com/d5/tengo v1.12.1
 	github.com/dfordsoft/golib v0.0.0-20180902042739-76ee6ab99bec
 	github.com/fsnotify/fsnotify v1.4.7
 	github.com/go-telegram-bot-api/telegram-bot-api v4.6.5-0.20181225215658-ec221ba9ea45+incompatible
-	github.com/golang/protobuf v0.0.0-20170613224224-e325f446bebc // indirect
 	github.com/google/gops v0.3.5
+	github.com/gopackage/ddp v0.0.0-20170117053602-652027933df4 // indirect
 	github.com/gopherjs/gopherjs v0.0.0-20180628210949-0892b62f0d9f // indirect
 	github.com/gorilla/schema v1.0.2
 	github.com/gorilla/websocket v1.4.0
@@ -20,17 +23,21 @@ require (
 	github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1 // indirect
 	github.com/kr/pretty v0.1.0 // indirect
 	github.com/labstack/echo/v4 v4.0.0
-	github.com/lrstanley/girc v0.0.0-20190102153329-c1e59a02f488
+	github.com/lrstanley/girc v0.0.0-20190210212025-51b8e096d398
 	github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5 // indirect
 	github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6 // indirect
+	github.com/matterbridge/Rocket.Chat.Go.SDK v0.0.0-20190210153444-cc9d05784d5d
+	github.com/matterbridge/go-whatsapp v0.0.1-0.20190301204034-f2f1b29d441b
 	github.com/matterbridge/go-xmpp v0.0.0-20180529212104-cd19799fba91
 	github.com/matterbridge/gomatrix v0.0.0-20190102230110-6f9631ca6dea
-	github.com/matterbridge/gozulipbot v0.0.0-20180507190239-b6bb12d33544
+	github.com/matterbridge/gozulipbot v0.0.0-20190212232658-7aa251978a18
 	github.com/matterbridge/logrus-prefixed-formatter v0.0.0-20180806162718-01618749af61
+	github.com/matterbridge/mautrix-whatsapp v0.0.0-20190301210046-3539cf52ed6e
 	github.com/mattermost/mattermost-server v5.5.0+incompatible
 	github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect
 	github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474 // indirect
 	github.com/mrexodia/wray v0.0.0-20160318003008-78a2c1f284ff // indirect
+	github.com/nelsonken/gomf v0.0.0-20180504123937-a9dd2f9deae9
 	github.com/nicksnyder/go-i18n v1.4.0 // indirect
 	github.com/nlopes/slack v0.5.0
 	github.com/onsi/ginkgo v1.6.0 // indirect
@@ -40,10 +47,9 @@ require (
 	github.com/peterhellberg/emojilib v0.0.0-20190124112554-c18758d55320
 	github.com/pkg/errors v0.8.0 // indirect
 	github.com/rs/xid v1.2.1
-	github.com/russross/blackfriday v2.0.0+incompatible
+	github.com/russross/blackfriday v1.5.2
 	github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca
 	github.com/shazow/ssh-chat v0.0.0-20190125184227-81d7e1686296
-	github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95 // indirect
 	github.com/sirupsen/logrus v1.3.0
 	github.com/smartystreets/assertions v0.0.0-20180803164922-886ec427f6b9 // indirect
 	github.com/smartystreets/goconvey v0.0.0-20180222194500-ef6db91d284a // indirect
@@ -61,9 +67,7 @@ require (
 	go.uber.org/atomic v1.3.2 // indirect
 	go.uber.org/multierr v1.1.0 // indirect
 	go.uber.org/zap v1.9.1 // indirect
-	golang.org/x/net v0.0.0-20180108090419-434ec0c7fe37 // indirect
+	golang.org/x/image v0.0.0-20190220214146-31aff87c08e9
-	golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f // indirect
-	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
 	gopkg.in/fsnotify.v1 v1.4.7 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect

go.sum

@@ -1,10 +1,13 @@
 github.com/42wim/go-gitter v0.0.0-20170828205020-017310c2d557 h1:IZtuWGfzQnKnCSu+vl8WGLhpVQ5Uvy3rlSwqXSg+sQg=
 github.com/42wim/go-gitter v0.0.0-20170828205020-017310c2d557/go.mod h1:jL0YSXMs/txjtGJ4PWrmETOk6KUHMDPMshgQZlTeB3Y=
+github.com/Baozisoftware/qrcode-terminal-go v0.0.0-20170407111555-c0650d8dff0f h1:2dk3eOnYllh+wUOuDhOoC2vUVoJF/5z478ryJ+wzEII=
+github.com/Baozisoftware/qrcode-terminal-go v0.0.0-20170407111555-c0650d8dff0f/go.mod h1:4a58ifQTEe2uwwsaqbh3i2un5/CBPg+At/qHpt18Tmk=
 github.com/BurntSushi/toml v0.0.0-20170318202913-d94612f9fc14 h1:v/zr4ns/4sSahF9KBm4Uc933bLsEEv7LuT63CJ019yo=
 github.com/BurntSushi/toml v0.0.0-20170318202913-d94612f9fc14/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/Jeffail/gabs v1.1.1 h1:V0uzR08Hj22EX8+8QMhyI9sX2hwRu+/RJhJUmnwda/E=
+github.com/Jeffail/gabs v1.1.1/go.mod h1:6xMvQMK4k33lb7GUUpaAPh6nKMmemQeg5d4gn7/bOXc=
 github.com/Philipp15b/go-steam v1.0.1-0.20180818081528-681bd9573329 h1:xZBoq249G9MSt+XuY7sVQzcfONJ6IQuwpCK+KAaOpnY=
 github.com/Philipp15b/go-steam v1.0.1-0.20180818081528-681bd9573329/go.mod h1:HuVM+sZFzumUdKPWiz+IlCMb4RdsKdT3T+nQBKL+sYg=
-github.com/alexcesaro/log v0.0.0-20150915221235-61e686294e58 h1:MkpmYfld/S8kXqTYI68DfL8/hHXjHogL120Dy00TIxc=
 github.com/alexcesaro/log v0.0.0-20150915221235-61e686294e58/go.mod h1:YNfsMyWSs+h+PaYkxGeMVmVCX75Zj/pqdjbu12ciCYE=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/bwmarrin/discordgo v0.19.0 h1:kMED/DB0NR1QhRcalb85w0Cu3Ep2OrGAqZH1R5awQiY=
@@ -12,6 +15,8 @@ github.com/bwmarrin/discordgo v0.19.0/go.mod h1:O9S4p+ofTFwB02em7jkpkV8M3R0/PUVO
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/d5/tengo v1.12.1 h1:libKkDM95CsZgYs6E5eiEaM9sbcw2EzJRSkr9o5NO4s=
+github.com/d5/tengo v1.12.1/go.mod h1:gsbjo7lBXzBIWBd6NQp1lRKqqiDDANqBOyhW8rTlFsY=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -19,16 +24,21 @@ github.com/dfordsoft/golib v0.0.0-20180902042739-76ee6ab99bec h1:JEUiu7P9smN7zgX
 github.com/dfordsoft/golib v0.0.0-20180902042739-76ee6ab99bec/go.mod h1:UGa5M2Sz/Uh13AMse4+RELKCDw7kqgqlTjeGae+7vUY=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/go-telegram-bot-api/telegram-bot-api v4.6.5-0.20181225215658-ec221ba9ea45+incompatible h1:i64CCJcSqkRIkm5OSdZQjZq84/gJsk2zNwHWIRYWlKE=
 github.com/go-telegram-bot-api/telegram-bot-api v4.6.5-0.20181225215658-ec221ba9ea45+incompatible/go.mod h1:qf9acutJ8cwBUhm1bqgz6Bei9/C/c93FPDljKWwsOgM=
-github.com/golang/protobuf v0.0.0-20170613224224-e325f446bebc h1:wdhDSKrkYy24mcfzuA3oYm58h0QkyXjwERCkzJDP5kA=
+github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
-github.com/golang/protobuf v0.0.0-20170613224224-e325f446bebc/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/google/gops v0.3.5 h1:SIWvPLiYvy5vMwjxB3rVFTE4QBhUFj2KKWr3Xm7CKhw=
 github.com/google/gops v0.3.5/go.mod h1:pMQgrscwEK/aUSW1IFSaBPbJX82FPHWaSoJw1axQfD0=
+github.com/gopackage/ddp v0.0.0-20170117053602-652027933df4 h1:4EZlYQIiyecYJlUbVkFXCXHz1QPhVXcHnQKAzBTPfQo=
+github.com/gopackage/ddp v0.0.0-20170117053602-652027933df4/go.mod h1:lEO7XoHJ/xNRBCxrn4h/CEB67h0kW1B0t4ooP2yrjUA=
 github.com/gopherjs/gopherjs v0.0.0-20180628210949-0892b62f0d9f h1:FDM3EtwZLyhW48YRiyqjivNlNZjAObv4xt4NnJaU+NQ=
 github.com/gopherjs/gopherjs v0.0.0-20180628210949-0892b62f0d9f/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
 github.com/gorilla/schema v1.0.2 h1:sAgNfOcNYvdDSrzGHVy9nzCQahG+qmsg+nE8dK85QRA=
 github.com/gorilla/schema v1.0.2/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlIO2eleU=
 github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
@@ -58,28 +68,37 @@ github.com/labstack/echo/v4 v4.0.0 h1:q1GH+caIXPP7H2StPIdzy/ez9CO0EepqYeUg6vi9SW
 github.com/labstack/echo/v4 v4.0.0/go.mod h1:tZv7nai5buKSg5h/8E6zz4LsD/Dqh9/91Mvs7Z5Zyno=
 github.com/labstack/gommon v0.2.8 h1:JvRqmeZcfrHC5u6uVleB4NxxNbzx6gpbJiQknDbKQu0=
 github.com/labstack/gommon v0.2.8/go.mod h1:/tj9csK2iPSBvn+3NLM9e52usepMtrd5ilFYA+wQNJ4=
-github.com/lrstanley/girc v0.0.0-20190102153329-c1e59a02f488 h1:dDEQN5oaa0WOzEiPDSbOugW/e2I/SWY98HYRdcwmGfY=
+github.com/lrstanley/girc v0.0.0-20190210212025-51b8e096d398 h1:a40kRmhA1p2XFJ6gqXfCExSyuDDCp/U9LA8ZY27u2Lk=
-github.com/lrstanley/girc v0.0.0-20190102153329-c1e59a02f488/go.mod h1:7cRs1SIBfKQ7e3Tam6GKTILSNHzR862JD0JpINaZoJk=
+github.com/lrstanley/girc v0.0.0-20190210212025-51b8e096d398/go.mod h1:7cRs1SIBfKQ7e3Tam6GKTILSNHzR862JD0JpINaZoJk=
 github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5 h1:AsEBgzv3DhuYHI/GiQh2HxvTP71HCCE9E/tzGUzGdtU=
 github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5/go.mod h1:c2mYKRyMb1BPkO5St0c/ps62L4S0W2NAkaTXj9qEI+0=
 github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6 h1:iOAVXzZyXtW408TMYejlUPo6BIn92HmOacWtIfNyYns=
 github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6/go.mod h1:sFlOUpQL1YcjhFVXhg1CG8ZASEs/Mf1oVb6H75JL/zg=
 github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/matterbridge/Rocket.Chat.Go.SDK v0.0.0-20190210153444-cc9d05784d5d h1:F+Sr+C0ojSlYQ37BLylQtSFmyQULe3jbAygcyXQ9mVs=
+github.com/matterbridge/Rocket.Chat.Go.SDK v0.0.0-20190210153444-cc9d05784d5d/go.mod h1:c6MxwqHD+0HvtAJjsHMIdPCiAwGiQwPRPTp69ACMg8A=
+github.com/matterbridge/go-whatsapp v0.0.1-0.20190301204034-f2f1b29d441b h1:cO6Z+yj4Ivq/ay/IxSrV90oSIW/SSXWLa+XHsiLKMrw=
+github.com/matterbridge/go-whatsapp v0.0.1-0.20190301204034-f2f1b29d441b/go.mod h1:dW19fYkkdUZsBAx7zv9fDh0n6NRqYIaKwB2JEBw8d0U=
 github.com/matterbridge/go-xmpp v0.0.0-20180529212104-cd19799fba91 h1:KzDEcy8eDbTx881giW8a6llsAck3e2bJvMyKvh1IK+k=
 github.com/matterbridge/go-xmpp v0.0.0-20180529212104-cd19799fba91/go.mod h1:ECDRehsR9TYTKCAsRS8/wLeOk6UUqDydw47ln7wG41Q=
 github.com/matterbridge/gomatrix v0.0.0-20190102230110-6f9631ca6dea h1:kaADGqpK4gGO2BpzEyJrBxq2Jc57Rsar4i2EUxcACUc=
 github.com/matterbridge/gomatrix v0.0.0-20190102230110-6f9631ca6dea/go.mod h1:+jWeaaUtXQbBRdKYWfjW6JDDYiI2XXE+3NnTjW5kg8g=
-github.com/matterbridge/gozulipbot v0.0.0-20180507190239-b6bb12d33544 h1:A8lLG3DAu75B5jITHs9z4JBmU6oCq1WiUNnDAmqKCZc=
+github.com/matterbridge/gozulipbot v0.0.0-20190212232658-7aa251978a18 h1:fLhwXtWGtfTgZVxHG1lcKjv+re7dRwyyuYFNu69xdho=
-github.com/matterbridge/gozulipbot v0.0.0-20180507190239-b6bb12d33544/go.mod h1:yAjnZ34DuDyPHMPHHjOsTk/FefW4JJjoMMCGt/8uuQA=
+github.com/matterbridge/gozulipbot v0.0.0-20190212232658-7aa251978a18/go.mod h1:yAjnZ34DuDyPHMPHHjOsTk/FefW4JJjoMMCGt/8uuQA=
 github.com/matterbridge/logrus-prefixed-formatter v0.0.0-20180806162718-01618749af61 h1:R/MgM/eUyRBQx2FiH6JVmXck8PaAuKfe2M1tWIzW7nE=
 github.com/matterbridge/logrus-prefixed-formatter v0.0.0-20180806162718-01618749af61/go.mod h1:iXGEotOvwI1R1SjLxRc+BF5rUORTMtE0iMZBT2lxqAU=
+github.com/matterbridge/mautrix-whatsapp v0.0.0-20190301210046-3539cf52ed6e h1:1NqciL8sz+0UYeFrd/UQlL8tJPhFxOBmg+a94DN2sJU=
+github.com/matterbridge/mautrix-whatsapp v0.0.0-20190301210046-3539cf52ed6e/go.mod h1:DrIFGcFumRlEW5k3PJjWGKPd4+w37d3SwOxlh1ZAL+4=
 github.com/mattermost/mattermost-server v5.5.0+incompatible h1:0wcLGgYtd+YImtLDPf2AOfpBHxbU4suATx+6XKw1XbU=
 github.com/mattermost/mattermost-server v5.5.0+incompatible/go.mod h1:5L6MjAec+XXQwMIt791Ganu45GKsSiM+I0tLR9wUj8Y=
-github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
-github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs=
+github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg=
+github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
 github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-isatty v0.0.5 h1:tHXDdz1cpzGaovsTB+TVB8q90WEokoVmfMqoVcrLUgw=
+github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4=
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
 github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
@@ -88,6 +107,8 @@ github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474 h1:oKIteT
 github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8=
 github.com/mrexodia/wray v0.0.0-20160318003008-78a2c1f284ff h1:HLGD5/9UxxfEuO9DtP8gnTmNtMxbPyhYltfxsITel8g=
 github.com/mrexodia/wray v0.0.0-20160318003008-78a2c1f284ff/go.mod h1:B8jLfIIPn2sKyWr0D7cL2v7tnrDD5z291s2Zypdu89E=
+github.com/nelsonken/gomf v0.0.0-20180504123937-a9dd2f9deae9 h1:mp6tU1r0xLostUGLkTspf/9/AiHuVD7ptyXhySkDEsE=
+github.com/nelsonken/gomf v0.0.0-20180504123937-a9dd2f9deae9/go.mod h1:A5SRAcpTemjGgIuBq6Kic2yHcoeUFWUinOAlMP/i9xo=
 github.com/nicksnyder/go-i18n v1.4.0 h1:AgLl+Yq7kg5OYlzCgu9cKTZOyI4tD/NgukKqLqC8E+I=
 github.com/nicksnyder/go-i18n v1.4.0/go.mod h1:HrK7VCrbOvQoUAQ7Vpy7i87N7JZZZ7R2xBGjv0j365Q=
 github.com/nlopes/slack v0.5.0 h1:NbIae8Kd0NpqaEI3iUrsuS0KbcEDhzhc939jLW5fNm0=
@@ -110,18 +131,20 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
-github.com/russross/blackfriday v2.0.0+incompatible h1:cBXrhZNUf9C+La9/YpS+UHpUT8YD6Td9ZMSU9APFcsk=
+github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo=
-github.com/russross/blackfriday v2.0.0+incompatible/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
+github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
+github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI=
 github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
 github.com/shazow/rateio v0.0.0-20150116013248-e8e00881e5c1 h1:Lx3BlDGFElJt4u/zKc9A3BuGYbQAGlEFyPuUA3jeMD0=
 github.com/shazow/rateio v0.0.0-20150116013248-e8e00881e5c1/go.mod h1:vt2jWY/3Qw1bIzle5thrJWucsLuuX9iUNnp20CqCciI=
 github.com/shazow/ssh-chat v0.0.0-20190125184227-81d7e1686296 h1:8RLq547MSVc6vhOuCl4Ca0TsAQknj6NX6ZLSZ3+xmio=
 github.com/shazow/ssh-chat v0.0.0-20190125184227-81d7e1686296/go.mod h1:1GLXsL4esywkpNId3v4QWuMf3THtWGitWvtQ/L3aSA4=
-github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95 h1:/vdW8Cb7EXrkqWGufVMES1OH2sU9gKVb2n9/1y5NMBY=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
-github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.3.0 h1:hI/7Q+DtNZ2kINb6qt/lS+IyXnHQe9e90POfeewL/ME=
 github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/skip2/go-qrcode v0.0.0-20190110000554-dc11ecdae0a9 h1:lpEzuenPuO1XNTeikEmvqYFcU37GVLl8SRNblzyvGBE=
+github.com/skip2/go-qrcode v0.0.0-20190110000554-dc11ecdae0a9/go.mod h1:PLPIyL7ikehBD1OAjmKKiOEhbvWyHGaNDjquXMcYABo=
 github.com/smartystreets/assertions v0.0.0-20180803164922-886ec427f6b9 h1:lXQ+j+KwZcbwrbgU0Rp4Eglg3EJLHbuZU3BbOqAGBmg=
 github.com/smartystreets/assertions v0.0.0-20180803164922-886ec427f6b9/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v0.0.0-20180222194500-ef6db91d284a h1:JSvGDIbmil4Ui/dDdFBExb7/cmkNjyX5F97oglmvCDo=
@@ -137,9 +160,7 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn
 github.com/spf13/viper v1.3.1 h1:5+8j8FTpnFV4nEImW/ofkzEt8VoOiLXxdYIDsB73T38=
 github.com/spf13/viper v1.3.1/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
@@ -174,19 +195,24 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/
 go.uber.org/zap v1.9.1 h1:XCJQEf3W6eZaVwhRBof6ImoYGJSITeKWsyeh3HFu/5o=
 go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16 h1:y6ce7gCWtnH+m3dCjzQ1PCuwl28DDIc3VNnvY29DlIA=
 golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20190130090550-b01c7a725664 h1:YbZJ76lQ1BqNhVe7dKTSB67wDrc2VPRR75IyGyyPDX8=
 golang.org/x/crypto v0.0.0-20190130090550-b01c7a725664/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/net v0.0.0-20180108090419-434ec0c7fe37 h1:BkNcmLtAVeWe9h5k0jt24CQgaG5vb4x/doFbAiEC/Ho=
+golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/net v0.0.0-20180108090419-434ec0c7fe37/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/crypto v0.0.0-20190222235706-ffb98f73852f h1:qWFY9ZxP3tfI37wYIs/MnIAqK0vlXp1xnYEa5HxFSSY=
-golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA=
+golang.org/x/crypto v0.0.0-20190222235706-ffb98f73852f/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/image v0.0.0-20190220214146-31aff87c08e9 h1:+vH8qNweCrORN49012OX3h0oWEXO3p+rRnpAGQinddk=
+golang.org/x/image v0.0.0-20190220214146-31aff87c08e9/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/net v0.0.0-20190110200230-915654e7eabc h1:Yx9JGxI1SBhVLFjpAkWMaO1TF+xyqtHLjZpvQboJGiM=
+golang.org/x/net v0.0.0-20190110200230-915654e7eabc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190129075346-302c3dd5f1cc h1:WiYx1rIFmx8c0mXAFtv5D/mHyKe1+jmuP7PViuwqwuQ=
 golang.org/x/sys v0.0.0-20190129075346-302c3dd5f1cc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190222171317-cd391775e71e h1:oF7qaQxUH6KzFdKN4ww7NpPdo53SZi4UlcksLrb2y/o=
+golang.org/x/sys v0.0.0-20190222171317-cd391775e71e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -200,3 +226,7 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkep
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+maunium.net/go/mauflag v1.0.0/go.mod h1:nLivPOpTpHnpzEh8jEdSL9UqO9+/KBJFmNRlwKfkPeA=
+maunium.net/go/maulogger/v2 v2.0.0/go.mod h1:Hbbkq3NV6jvJodByZu1mgEF3fpT7Kz9z0MjEZ3/BusI=
+maunium.net/go/mautrix v0.1.0-alpha.3/go.mod h1:GTVu6WDHR+98DKOrYetWsXorvUeKQV3jsSWO6ScbuFI=
+maunium.net/go/mautrix-appservice v0.1.0-alpha.3/go.mod h1:wOnWOIuprYad7ly12rHIo3JLCPh4jwvx1prVrAB9RhM=

matterbridge.go

@@ -15,48 +15,71 @@ import (
 )
 
 var (
-	version = "1.13.1"
+	version = "1.14.0"
 	githash string
+
+	flagConfig  = flag.String("conf", "matterbridge.toml", "config file")
+	flagDebug   = flag.Bool("debug", false, "enable debug")
+	flagVersion = flag.Bool("version", false, "show version")
+	flagGops    = flag.Bool("gops", false, "enable gops agent")
 )
 
 func main() {
-	logrus.SetFormatter(&prefixed.TextFormatter{PrefixPadding: 13, DisableColors: true, FullTimestamp: true})
-	flog := logrus.WithFields(logrus.Fields{"prefix": "main"})
-	flagConfig := flag.String("conf", "matterbridge.toml", "config file")
-	flagDebug := flag.Bool("debug", false, "enable debug")
-	flagVersion := flag.Bool("version", false, "show version")
-	flagGops := flag.Bool("gops", false, "enable gops agent")
 	flag.Parse()
-	if *flagGops {
-		if err := agent.Listen(agent.Options{}); err != nil {
-			flog.Errorf("failed to start gops agent: %#v", err)
-		} else {
-			defer agent.Close()
-		}
-	}
 	if *flagVersion {
 		fmt.Printf("version: %s %s\n", version, githash)
 		return
 	}
-	if *flagDebug || os.Getenv("DEBUG") == "1" {
+
-		logrus.SetFormatter(&prefixed.TextFormatter{PrefixPadding: 13, DisableColors: true, FullTimestamp: false, ForceFormatting: true})
+	rootLogger := setupLogger()
-		flog.Info("Enabling debug")
+	logger := rootLogger.WithFields(logrus.Fields{"prefix": "main"})
-		logrus.SetLevel(logrus.DebugLevel)
+
+	if *flagGops {
+		if err := agent.Listen(agent.Options{}); err != nil {
+			logger.Errorf("Failed to start gops agent: %#v", err)
+		} else {
+			defer agent.Close()
+		}
 	}
-	flog.Printf("Running version %s %s", version, githash)
+
+	logger.Printf("Running version %s %s", version, githash)
 	if strings.Contains(version, "-dev") {
-		flog.Println("WARNING: THIS IS A DEVELOPMENT VERSION. Things may break.")
+		logger.Println("WARNING: THIS IS A DEVELOPMENT VERSION. Things may break.")
 	}
-	cfg := config.NewConfig(*flagConfig)
+
+	cfg := config.NewConfig(rootLogger, *flagConfig)
 	cfg.BridgeValues().General.Debug = *flagDebug
-	r, err := gateway.NewRouter(cfg, bridgemap.FullMap)
+
+	r, err := gateway.NewRouter(rootLogger, cfg, bridgemap.FullMap)
 	if err != nil {
-		flog.Fatalf("Starting gateway failed: %s", err)
+		logger.Fatalf("Starting gateway failed: %s", err)
 	}
-	err = r.Start()
+	if err = r.Start(); err != nil {
-	if err != nil {
+		logger.Fatalf("Starting gateway failed: %s", err)
-		flog.Fatalf("Starting gateway failed: %s", err)
 	}
-	flog.Printf("Gateway(s) started succesfully. Now relaying messages")
+	logger.Printf("Gateway(s) started succesfully. Now relaying messages")
 	select {}
 }
+
+func setupLogger() *logrus.Logger {
+	logger := &logrus.Logger{
+		Out: os.Stdout,
+		Formatter: &prefixed.TextFormatter{
+			PrefixPadding: 13,
+			DisableColors: true,
+			FullTimestamp: true,
+		},
+		Level: logrus.InfoLevel,
+	}
+	if *flagDebug || os.Getenv("DEBUG") == "1" {
+		logger.Formatter = &prefixed.TextFormatter{
+			PrefixPadding:   13,
+			DisableColors:   true,
+			FullTimestamp:   false,
+			ForceFormatting: true,
+		}
+		logger.Level = logrus.DebugLevel
+		logger.WithFields(logrus.Fields{"prefix": "main"}).Info("Enabling debug logging.")
+	}
+	return logger
+}

matterbridge.toml.sample

@@ -102,6 +102,7 @@ ColorNicks=false
 RunCommands=["PRIVMSG user hello","PRIVMSG chanserv something"]
 
 #Nicks you want to ignore. 
+#Regular expressions supported
 #Messages from those users will not be sent to other bridges.
 #OPTIONAL
 IgnoreNicks="ircspammer1 ircspammer2"
@@ -129,6 +130,17 @@ ReplaceMessages=[ ["cat","dog"] ]
 #optional (default empty)
 ReplaceNicks=[ ["user--","user"] ]
 
+#Extractnicks is used to for example rewrite messages from other relaybots
+#See https://github.com/42wim/matterbridge/issues/713 and https://github.com/42wim/matterbridge/issues/466
+#some examples:
+#this replaces a message like "Relaybot: <relayeduser> something interesting" to "relayeduser: something interesting"
+#ExtractNicks=[ [ "Relaybot", "<(.*?)>\\s+" ] ]
+#you can use multiple entries for multiplebots
+#this also replaces a message like "otherbot: (relayeduser) something else" to "relayeduser: something else"
+#ExtractNicks=[ [ "Relaybot", "<(.*?)>\\s+" ],[ "otherbot","\\((.*?)\\)\\s+" ]
+#OPTIONAL (default empty)
+ExtractNicks=[ ["otherbot","<(.*?)>\\s+" ] ]
+
 #extra label that can be used in the RemoteNickFormat
 #optional (default empty)
 Label=""
@@ -139,7 +151,7 @@ Label=""
 RemoteNickFormat="[{PROTOCOL}] <{NICK}> "
 
 #Enable to show users joins/parts from other bridges 
-#Currently works for messages from the following bridges: irc, mattermost, slack
+#Currently works for messages from the following bridges: irc, mattermost, slack, discord
 #OPTIONAL (default false)
 ShowJoinPart=false
 
@@ -196,6 +208,7 @@ SkipTLSVerify=true
 ## Settings below can be reloaded by editing the file
 
 #Nicks you want to ignore. 
+#Regular expressions supported
 #Messages from those users will not be sent to other bridges.
 #OPTIONAL
 IgnoreNicks="ircspammer1 ircspammer2"
@@ -223,6 +236,17 @@ ReplaceMessages=[ ["cat","dog"] ]
 #OPTIONAL (default empty)
 ReplaceNicks=[ ["user--","user"] ]
 
+#Extractnicks is used to for example rewrite messages from other relaybots
+#See https://github.com/42wim/matterbridge/issues/713 and https://github.com/42wim/matterbridge/issues/466
+#some examples:
+#this replaces a message like "Relaybot: <relayeduser> something interesting" to "relayeduser: something interesting"
+#ExtractNicks=[ [ "Relaybot", "<(.*?)>\\s+" ] ]
+#you can use multiple entries for multiplebots
+#this also replaces a message like "otherbot: (relayeduser) something else" to "relayeduser: something else"
+#ExtractNicks=[ [ "Relaybot", "<(.*?)>\\s+" ],[ "otherbot","\\((.*?)\\)\\s+" ]
+#OPTIONAL (default empty)
+ExtractNicks=[ ["otherbot","<(.*?)>\\s+" ] ]
+
 #extra label that can be used in the RemoteNickFormat
 #optional (default empty)
 Label=""
@@ -232,7 +256,7 @@ Label=""
 RemoteNickFormat="[{PROTOCOL}] <{NICK}> "
 
 #Enable to show users joins/parts from other bridges 
-#Currently works for messages from the following bridges: irc, mattermost, slack
+#Currently works for messages from the following bridges: irc, mattermost, slack, discord
 #OPTIONAL (default false)
 ShowJoinPart=false
 
@@ -276,6 +300,7 @@ Nick="yourlogin"
 ## Settings below can be reloaded by editing the file
 
 #Nicks you want to ignore. 
+#Regular expressions supported
 #Messages from those users will not be sent to other bridges.
 #OPTIONAL
 IgnoreNicks="spammer1 spammer2"
@@ -303,6 +328,17 @@ ReplaceMessages=[ ["cat","dog"] ]
 #optional (default empty)
 ReplaceNicks=[ ["user--","user"] ]
 
+#Extractnicks is used to for example rewrite messages from other relaybots
+#See https://github.com/42wim/matterbridge/issues/713 and https://github.com/42wim/matterbridge/issues/466
+#some examples:
+#this replaces a message like "Relaybot: <relayeduser> something interesting" to "relayeduser: something interesting"
+#ExtractNicks=[ [ "Relaybot", "<(.*?)>\\s+" ] ]
+#you can use multiple entries for multiplebots
+#this also replaces a message like "otherbot: (relayeduser) something else" to "relayeduser: something else"
+#ExtractNicks=[ [ "Relaybot", "<(.*?)>\\s+" ],[ "otherbot","\\((.*?)\\)\\s+" ]
+#OPTIONAL (default empty)
+ExtractNicks=[ ["otherbot","<(.*?)>\\s+" ] ]
+
 #extra label that can be used in the RemoteNickFormat
 #optional (default empty)
 Label=""
@@ -312,7 +348,7 @@ Label=""
 RemoteNickFormat="[{PROTOCOL}/{BRIDGE}] <{NICK}> "
 
 #Enable to show users joins/parts from other bridges 
-#Currently works for messages from the following bridges: irc, mattermost, slack
+#Currently works for messages from the following bridges: irc, mattermost, slack, discord
 #OPTIONAL (default false)
 ShowJoinPart=false
 
@@ -416,6 +452,7 @@ EditDisable=false
 EditSuffix=" (edited)"
 
 #Nicks you want to ignore. 
+#Regular expressions supported
 #Messages from those users will not be sent to other bridges.
 #OPTIONAL
 IgnoreNicks="ircspammer1 ircspammer2"
@@ -443,6 +480,17 @@ ReplaceMessages=[ ["cat","dog"] ]
 #optional (default empty)
 ReplaceNicks=[ ["user--","user"] ]
 
+#Extractnicks is used to for example rewrite messages from other relaybots
+#See https://github.com/42wim/matterbridge/issues/713 and https://github.com/42wim/matterbridge/issues/466
+#some examples:
+#this replaces a message like "Relaybot: <relayeduser> something interesting" to "relayeduser: something interesting"
+#ExtractNicks=[ [ "Relaybot", "<(.*?)>\\s+" ] ]
+#you can use multiple entries for multiplebots
+#this also replaces a message like "otherbot: (relayeduser) something else" to "relayeduser: something else"
+#ExtractNicks=[ [ "Relaybot", "<(.*?)>\\s+" ],[ "otherbot","\\((.*?)\\)\\s+" ]
+#OPTIONAL (default empty)
+ExtractNicks=[ ["otherbot","<(.*?)>\\s+" ] ]
+
 #extra label that can be used in the RemoteNickFormat
 #optional (default empty)
 Label=""
@@ -452,7 +500,7 @@ Label=""
 RemoteNickFormat="[{PROTOCOL}] <{NICK}> "
 
 #Enable to show users joins/parts from other bridges 
-#Currently works for messages from the following bridges: irc, mattermost, slack
+#Currently works for messages from the following bridges: irc, mattermost, slack, discord
 #OPTIONAL (default false)
 ShowJoinPart=false
 
@@ -491,6 +539,7 @@ Token="Yourtokenhere"
 ## Settings below can be reloaded by editing the file
 
 #Nicks you want to ignore. 
+#Regular expressions supported
 #Messages from those users will not be sent to other bridges.
 #OPTIONAL
 IgnoreNicks="ircspammer1 ircspammer2"
@@ -518,6 +567,17 @@ ReplaceMessages=[ ["cat","dog"] ]
 #optional (default empty)
 ReplaceNicks=[ ["user--","user"] ]
 
+#Extractnicks is used to for example rewrite messages from other relaybots
+#See https://github.com/42wim/matterbridge/issues/713 and https://github.com/42wim/matterbridge/issues/466
+#some examples:
+#this replaces a message like "Relaybot: <relayeduser> something interesting" to "relayeduser: something interesting"
+#ExtractNicks=[ [ "Relaybot", "<(.*?)>\\s+" ] ]
+#you can use multiple entries for multiplebots
+#this also replaces a message like "otherbot: (relayeduser) something else" to "relayeduser: something else"
+#ExtractNicks=[ [ "Relaybot", "<(.*?)>\\s+" ],[ "otherbot","\\((.*?)\\)\\s+" ]
+#OPTIONAL (default empty)
+ExtractNicks=[ ["otherbot","<(.*?)>\\s+" ] ]
+
 #extra label that can be used in the RemoteNickFormat
 #optional (default empty)
 Label=""
@@ -527,7 +587,7 @@ Label=""
 RemoteNickFormat="[{PROTOCOL}] <{NICK}> "
 
 #Enable to show users joins/parts from other bridges 
-#Currently works for messages from the following bridges: irc, mattermost, slack
+#Currently works for messages from the following bridges: irc, mattermost, slack, discord
 #OPTIONAL (default false)
 ShowJoinPart=false
 
@@ -557,6 +617,10 @@ ShowTopicChange=false
 #REQUIRED (when not using webhooks)
 Token="yourslacktoken"
 
+#Extra slack specific debug info, warning this generates a lot of output.
+#OPTIONAL (default false)
+Debug="false"
+
 #### Settings for webhook matterbridge.
 #NOT RECOMMENDED TO USE INCOMING/OUTGOING WEBHOOK. USE SLACK API
 #AND DEDICATED BOT USER WHEN POSSIBLE!
@@ -609,6 +673,7 @@ EditSuffix=" (edited)"
 PrefixMessagesWithNick=false
 
 #Nicks you want to ignore. 
+#Regular expressions supported
 #Messages from those users will not be sent to other bridges.
 #OPTIONAL
 IgnoreNicks="ircspammer1 ircspammer2"
@@ -636,6 +701,17 @@ ReplaceMessages=[ ["cat","dog"] ]
 #optional (default empty)
 ReplaceNicks=[ ["user--","user"] ]
 
+#Extractnicks is used to for example rewrite messages from other relaybots
+#See https://github.com/42wim/matterbridge/issues/713 and https://github.com/42wim/matterbridge/issues/466
+#some examples:
+#this replaces a message like "Relaybot: <relayeduser> something interesting" to "relayeduser: something interesting"
+#ExtractNicks=[ [ "Relaybot", "<(.*?)>\\s+" ] ]
+#you can use multiple entries for multiplebots
+#this also replaces a message like "otherbot: (relayeduser) something else" to "relayeduser: something else"
+#ExtractNicks=[ [ "Relaybot", "<(.*?)>\\s+" ],[ "otherbot","\\((.*?)\\)\\s+" ]
+#OPTIONAL (default empty)
+ExtractNicks=[ ["otherbot","<(.*?)>\\s+" ] ]
+
 #extra label that can be used in the RemoteNickFormat
 #optional (default empty)
 Label=""
@@ -645,7 +721,7 @@ Label=""
 RemoteNickFormat="[{PROTOCOL}] <{NICK}> "
 
 #Enable to show users joins/parts from other bridges 
-#Currently works for messages from the following bridges: irc, mattermost, slack
+#Currently works for messages from the following bridges: irc, mattermost, slack, discord
 #OPTIONAL (default false)
 ShowJoinPart=false
 
@@ -704,10 +780,14 @@ Server="yourservername"
 #OPTIONAL (default false)
 ShowEmbeds=false
 
-#Shows the username (minus the discriminator) instead of the server nickname
+#Shows the username instead of the server nickname
 #OPTIONAL (default false)
 UseUserName=false
 
+#Show #xxxx discriminator with UseUserName
+#OPTIONAL (default false)
+UseDiscriminator=false
+
 #Specify WebhookURL. If given, will relay messages using the Webhook, which gives a better look to messages.
 #This only works if you have one discord channel, if you have multiple discord channels you'll have to specify it in the gateway config
 #OPTIONAL (default empty)
@@ -722,6 +802,7 @@ EditDisable=false
 EditSuffix=" (edited)"
 
 #Nicks you want to ignore. 
+#Regular expressions supported
 #Messages from those users will not be sent to other bridges.
 #OPTIONAL
 IgnoreNicks="ircspammer1 ircspammer2"
@@ -749,6 +830,17 @@ ReplaceMessages=[ ["cat","dog"] ]
 #optional (default empty)
 ReplaceNicks=[ ["user--","user"] ]
 
+#Extractnicks is used to for example rewrite messages from other relaybots
+#See https://github.com/42wim/matterbridge/issues/713 and https://github.com/42wim/matterbridge/issues/466
+#some examples:
+#this replaces a message like "Relaybot: <relayeduser> something interesting" to "relayeduser: something interesting"
+#ExtractNicks=[ [ "Relaybot", "<(.*?)>\\s+" ] ]
+#you can use multiple entries for multiplebots
+#this also replaces a message like "otherbot: (relayeduser) something else" to "relayeduser: something else"
+#ExtractNicks=[ [ "Relaybot", "<(.*?)>\\s+" ],[ "otherbot","\\((.*?)\\)\\s+" ]
+#OPTIONAL (default empty)
+ExtractNicks=[ ["otherbot","<(.*?)>\\s+" ] ]
+
 #extra label that can be used in the RemoteNickFormat
 #optional (default empty)
 Label=""
@@ -758,7 +850,7 @@ Label=""
 RemoteNickFormat="[{PROTOCOL}] <{NICK}> "
 
 #Enable to show users joins/parts from other bridges 
-#Currently works for messages from the following bridges: irc, mattermost, slack
+#Currently works for messages from the following bridges: irc, mattermost, slack, discord
 #OPTIONAL (default false)
 ShowJoinPart=false
 
@@ -821,6 +913,11 @@ QuoteDisable=false
 #OPTIONAL (default "{MESSAGE} (re @{QUOTENICK}: {QUOTEMESSAGE})")
 QuoteFormat="{MESSAGE} (re @{QUOTENICK}: {QUOTEMESSAGE})"
 
+#Convert WebP images to PNG before upload.
+#https://github.com/42wim/matterbridge/issues/398
+#OPTIONAL (default false)
+MediaConvertWebPToPNG=false
+
 #Disable sending of edits to other bridges
 #OPTIONAL (default false)
 EditDisable=false
@@ -830,6 +927,7 @@ EditDisable=false
 EditSuffix=" (edited)"
 
 #Nicks you want to ignore. 
+#Regular expressions supported
 #Messages from those users will not be sent to other bridges.
 #OPTIONAL
 IgnoreNicks="spammer1 spammer2"
@@ -857,6 +955,17 @@ ReplaceMessages=[ ["cat","dog"] ]
 #optional (default empty)
 ReplaceNicks=[ ["user--","user"] ]
 
+#Extractnicks is used to for example rewrite messages from other relaybots
+#See https://github.com/42wim/matterbridge/issues/713 and https://github.com/42wim/matterbridge/issues/466
+#some examples:
+#this replaces a message like "Relaybot: <relayeduser> something interesting" to "relayeduser: something interesting"
+#ExtractNicks=[ [ "Relaybot", "<(.*?)>\\s+" ] ]
+#you can use multiple entries for multiplebots
+#this also replaces a message like "otherbot: (relayeduser) something else" to "relayeduser: something else"
+#ExtractNicks=[ [ "Relaybot", "<(.*?)>\\s+" ],[ "otherbot","\\((.*?)\\)\\s+" ]
+#OPTIONAL (default empty)
+ExtractNicks=[ ["otherbot","<(.*?)>\\s+" ] ]
+
 #extra label that can be used in the RemoteNickFormat
 #optional (default empty)
 Label=""
@@ -870,7 +979,7 @@ Label=""
 RemoteNickFormat="[{PROTOCOL}] <{NICK}> "
 
 #Enable to show users joins/parts from other bridges 
-#Currently works for messages from the following bridges: irc, mattermost, slack
+#Currently works for messages from the following bridges: irc, mattermost, slack, discord
 #OPTIONAL (default false)
 ShowJoinPart=false
 
@@ -893,6 +1002,22 @@ ShowTopicChange=false
 #REQUIRED
 
 [rocketchat.rockme]
+#The rocketchat hostname. (prefix it with http or https)
+#REQUIRED (when not using webhooks)
+Server="https://yourrocketchatserver.domain.com:443"
+
+#login/pass of your bot. 
+#login needs to be the login with email address! user@domain.com
+#Use a dedicated user for this and not your own! 
+#REQUIRED (when not using webhooks)
+Login="yourlogin@domain.com"
+Password="yourpass"
+
+#### Settings for webhook matterbridge.
+#USE DEDICATED BOT USER WHEN POSSIBLE! This allows you to use advanced features like message editing/deleting and uploads
+#You don't need to configure this, if you have configured the settings 
+#above.
+
 #Url is your incoming webhook url as specified in rocketchat
 #Read #https://rocket.chat/docs/administrator-guides/integrations/#how-to-create-a-new-incoming-webhook
 #See administration - integrations - new integration - incoming webhook
@@ -917,6 +1042,8 @@ NoTLS=false
 #OPTIONAL (default false)
 SkipTLSVerify=true
 
+#### End settings for webhook matterbridge.
+
 ## RELOADABLE SETTINGS
 ## Settings below can be reloaded by editing the file
 
@@ -928,6 +1055,7 @@ SkipTLSVerify=true
 PrefixMessagesWithNick=false
 
 #Nicks you want to ignore. 
+#Regular expressions supported
 #Messages from those users will not be sent to other bridges.
 #OPTIONAL
 IgnoreNicks="ircspammer1 ircspammer2"
@@ -955,6 +1083,17 @@ ReplaceMessages=[ ["cat","dog"] ]
 #optional (default empty)
 ReplaceNicks=[ ["user--","user"] ]
 
+#Extractnicks is used to for example rewrite messages from other relaybots
+#See https://github.com/42wim/matterbridge/issues/713 and https://github.com/42wim/matterbridge/issues/466
+#some examples:
+#this replaces a message like "Relaybot: <relayeduser> something interesting" to "relayeduser: something interesting"
+#ExtractNicks=[ [ "Relaybot", "<(.*?)>\\s+" ] ]
+#you can use multiple entries for multiplebots
+#this also replaces a message like "otherbot: (relayeduser) something else" to "relayeduser: something else"
+#ExtractNicks=[ [ "Relaybot", "<(.*?)>\\s+" ],[ "otherbot","\\((.*?)\\)\\s+" ]
+#OPTIONAL (default empty)
+ExtractNicks=[ ["otherbot","<(.*?)>\\s+" ] ]
+
 #extra label that can be used in the RemoteNickFormat
 #optional (default empty)
 Label=""
@@ -964,7 +1103,7 @@ Label=""
 RemoteNickFormat="[{PROTOCOL}] <{NICK}> "
 
 #Enable to show users joins/parts from other bridges 
-#Currently works for messages from the following bridges: irc, mattermost, slack
+#Currently works for messages from the following bridges: irc, mattermost, slack, discord
 #OPTIONAL (default false)
 ShowJoinPart=false
 
@@ -1014,6 +1153,7 @@ NoHomeServerSuffix=false
 PrefixMessagesWithNick=false
 
 #Nicks you want to ignore. 
+#Regular expressions supported
 #Messages from those users will not be sent to other bridges.
 #OPTIONAL
 IgnoreNicks="spammer1 spammer2"
@@ -1041,6 +1181,17 @@ ReplaceMessages=[ ["cat","dog"] ]
 #optional (default empty)
 ReplaceNicks=[ ["user--","user"] ]
 
+#Extractnicks is used to for example rewrite messages from other relaybots
+#See https://github.com/42wim/matterbridge/issues/713 and https://github.com/42wim/matterbridge/issues/466
+#some examples:
+#this replaces a message like "Relaybot: <relayeduser> something interesting" to "relayeduser: something interesting"
+#ExtractNicks=[ [ "Relaybot", "<(.*?)>\\s+" ] ]
+#you can use multiple entries for multiplebots
+#this also replaces a message like "otherbot: (relayeduser) something else" to "relayeduser: something else"
+#ExtractNicks=[ [ "Relaybot", "<(.*?)>\\s+" ],[ "otherbot","\\((.*?)\\)\\s+" ]
+#OPTIONAL (default empty)
+ExtractNicks=[ ["otherbot","<(.*?)>\\s+" ] ]
+
 #extra label that can be used in the RemoteNickFormat
 #optional (default empty)
 Label=""
@@ -1050,7 +1201,7 @@ Label=""
 RemoteNickFormat="[{PROTOCOL}] <{NICK}> "
 
 #Enable to show users joins/parts from other bridges 
-#Currently works for messages from the following bridges: irc, mattermost, slack
+#Currently works for messages from the following bridges: irc, mattermost, slack, discord
 #OPTIONAL (default false)
 ShowJoinPart=false
 
@@ -1094,6 +1245,7 @@ Authcode="ABCE12"
 PrefixMessagesWithNick=false
 
 #Nicks you want to ignore. 
+#Regular expressions supported
 #Messages from those users will not be sent to other bridges.
 #OPTIONAL
 IgnoreNicks="spammer1 spammer2"
@@ -1121,6 +1273,17 @@ ReplaceMessages=[ ["cat","dog"] ]
 #optional (default empty)
 ReplaceNicks=[ ["user--","user"] ]
 
+#Extractnicks is used to for example rewrite messages from other relaybots
+#See https://github.com/42wim/matterbridge/issues/713 and https://github.com/42wim/matterbridge/issues/466
+#some examples:
+#this replaces a message like "Relaybot: <relayeduser> something interesting" to "relayeduser: something interesting"
+#ExtractNicks=[ [ "Relaybot", "<(.*?)>\\s+" ] ]
+#you can use multiple entries for multiplebots
+#this also replaces a message like "otherbot: (relayeduser) something else" to "relayeduser: something else"
+#ExtractNicks=[ [ "Relaybot", "<(.*?)>\\s+" ],[ "otherbot","\\((.*?)\\)\\s+" ]
+#OPTIONAL (default empty)
+ExtractNicks=[ ["otherbot","<(.*?)>\\s+" ] ]
+
 #extra label that can be used in the RemoteNickFormat
 #optional (default empty)
 Label=""
@@ -1130,7 +1293,7 @@ Label=""
 RemoteNickFormat="[{PROTOCOL}] <{NICK}> "
 
 #Enable to show users joins/parts from other bridges 
-#Currently works for messages from the following bridges: irc, mattermost, slack
+#Currently works for messages from the following bridges: irc, mattermost, slack, discord
 #OPTIONAL (default false)
 ShowJoinPart=false
 
@@ -1144,10 +1307,45 @@ StripNick=false
 #OPTIONAL (default false)
 ShowTopicChange=false
 
+
+
 ###################################################################
-#zulip section
+#
+# WhatsApp
+#
 ###################################################################
+
+[whatsapp.bridge]
+
+# Number you will use as a relay bot. Tip: Get some disposable sim card, don't rely on your own number.
+Number="+48111222333"
+
+# First time that you login you will need to scan QR code, then credentials willl be saved in a session file
+# If you won't set SessionFile then you will need to scan QR code on every restart
+# optional (by default the session is stored only in memory, till restarting matterbridge)
+SessionFile="session-48111222333.gob"
+
+# If your terminal is white we need to invert QR code in order for it to be scanned properly
+# optional (default false)
+QrOnWhiteTerminal=true
+
+# Messages will be seen by other WhatsApp contacts as coming from the bridge. Original nick will be part of the message.
+RemoteNickFormat="@{NICK}: "
+
+# extra label that can be used in the RemoteNickFormat
+# optional (default empty)
+Label="Organization"
+
+
+
+###################################################################
+#
+# zulip
+#
+###################################################################
+
 [zulip]
+
 #You can configure multiple servers "[zulip.name]" or "[zulip.name2]"
 #In this example we use [zulip.streamchat]
 #REQUIRED
@@ -1166,14 +1364,11 @@ Login="yourbot-bot@yourserver.zulipchat.com"
 #REQUIRED 
 Server="https://yourserver.zulipchat.com"
 
-#Topic of the messages matterbridge will use
-#OPTIONAL (default "matterbridge")
-Topic="matterbridge"
-
 ## RELOADABLE SETTINGS
 ## Settings below can be reloaded by editing the file
 
 #Nicks you want to ignore. 
+#Regular expressions supported
 #Messages from those users will not be sent to other bridges.
 #OPTIONAL
 IgnoreNicks="spammer1 spammer2"
@@ -1201,6 +1396,17 @@ ReplaceMessages=[ ["cat","dog"] ]
 #optional (default empty)
 ReplaceNicks=[ ["user--","user"] ]
 
+#Extractnicks is used to for example rewrite messages from other relaybots
+#See https://github.com/42wim/matterbridge/issues/713 and https://github.com/42wim/matterbridge/issues/466
+#some examples:
+#this replaces a message like "Relaybot: <relayeduser> something interesting" to "relayeduser: something interesting"
+#ExtractNicks=[ [ "Relaybot", "<(.*?)>\\s+" ] ]
+#you can use multiple entries for multiplebots
+#this also replaces a message like "otherbot: (relayeduser) something else" to "relayeduser: something else"
+#ExtractNicks=[ [ "Relaybot", "<(.*?)>\\s+" ],[ "otherbot","\\((.*?)\\)\\s+" ]
+#OPTIONAL (default empty)
+ExtractNicks=[ ["otherbot","<(.*?)>\\s+" ] ]
+
 #extra label that can be used in the RemoteNickFormat
 #optional (default empty)
 Label=""
@@ -1210,7 +1416,7 @@ Label=""
 RemoteNickFormat="[{PROTOCOL}] <{NICK}> "
 
 #Enable to show users joins/parts from other bridges 
-#Currently works for messages from the following bridges: irc, mattermost, slack
+#Currently works for messages from the following bridges: irc, mattermost, slack, discord
 #OPTIONAL (default false)
 ShowJoinPart=false
 
@@ -1321,6 +1527,29 @@ MediaDownloadBlacklist=[".html$",".htm$"]
 #OPTIONAL (default false)
 IgnoreFailureOnStart=false
 
+
+#TengoModifyMessage allows you to specify the location of a tengo (https://github.com/d5/tengo/) script.
+#This script will receive every incoming message and can be used to modify the Username and the Text of that message.
+#The script will have the following global variables:
+#to modify: msgUsername and msgText
+#to read: msgChannel and msgAccount
+#
+#The script is reloaded on every message, so you can modify the script on the fly.
+#
+#Example script can be found in https://github.com/42wim/matterbridge/tree/master/gateway/bench.tengo
+#and https://github.com/42wim/matterbridge/tree/master/contrib/example.tengo
+#
+#The example below will check if the text contains blah and if so, it'll replace the text and the username of that message.
+#text := import("text")
+#if text.re_match("blah",msgText) {
+#    msgText="replaced by this"
+#    msgUsername="fakeuser"
+#}
+#More information about tengo on: https://github.com/d5/tengo/blob/master/docs/tutorial.md and
+#https://github.com/d5/tengo/blob/master/docs/stdlib.md
+#OPTIONAL (default empty)
+TengoModifyMessage="example.tengo"
+
 ###################################################################
 #Gateway configuration
 ###################################################################
@@ -1342,36 +1571,41 @@ name="gateway1"
 ##OPTIONAL (default false)
 enable=true
 
-    #[[gateway.in]] specifies the account and channels we will receive messages from.
+    # [[gateway.in]] specifies the account and channels we will receive messages from.
-    #The following example bridges between mattermost and irc
+    # The following example bridges between mattermost and irc
     [[gateway.in]]
 
-    #account specified above
+    # account specified above
-    #REQUIRED
+    # REQUIRED
     account="irc.freenode"
-    #channel to connect on that account
+
-    #How to specify them for the different bridges:
+    # channel to connect on that account
+    # How to specify them for the different bridges:
     #
-    #irc        - #channel (# is required) (this needs to be lowercase!)
+    # irc        - #channel (# is required) (this needs to be lowercase!)
-    #mattermost - channel (the channel name as seen in the URL, not the displayname)
+    # mattermost - channel (the channel name as seen in the URL, not the displayname)
-    #gitter     - username/room 
+    # gitter     - username/room
-    #xmpp       - channel
+    # xmpp       - channel
-    #slack      - channel (without the #)
+    # slack      - channel (without the #)
-    #           - ID:C123456 (where C123456 is the channel ID) does not work with webhook
+    #            - ID:C123456 (where C123456 is the channel ID) does not work with webhook
-    #discord    - channel (without the #)
+    # discord    - channel (without the #)
-    #           - ID:123456789 (where 123456789 is the channel ID) 
+    #            - ID:123456789 (where 123456789 is the channel ID)
     #               (https://github.com/42wim/matterbridge/issues/57)
-    #telegram   - chatid (a large negative number, eg -123456789)
+    # telegram   - chatid (a large negative number, eg -123456789)
     #             see (https://www.linkedin.com/pulse/telegram-bots-beginners-marco-frau)
-    #hipchat    - id_channel (see https://www.hipchat.com/account/xmpp for the correct channel)
+    # hipchat    - id_channel (see https://www.hipchat.com/account/xmpp for the correct channel)
-    #rocketchat - #channel (# is required (also needed for private channels!)
+    # rocketchat - #channel (# is required (also needed for private channels!)
-    #matrix     - #channel:server (eg #yourchannel:matrix.org) 
+    # matrix     - #channel:server (eg #yourchannel:matrix.org)
-    #           - encrypted rooms are not supported in matrix
+    #            - encrypted rooms are not supported in matrix
-    #steam      - chatid (a large number). 
+    # steam      - chatid (a large number).
     #             The number in the URL when you click "enter chat room" in the browser
-    #zulip      - stream (without the #)
+    # whatsapp   - 48111222333-123455678999@g.us A unique group JID;
+    #              if you specify an empty string bridge will list all the possibilities
+    #            - "Group Name" if you specify a group name the bridge will hint its JID to specify
+    #              as group names might change in time and contain weird emoticons
+    # zulip      - stream/topic:topicname (without the #)
     #                  
-    #REQUIRED
+    # REQUIRED
     channel="#testing"
 
         #OPTIONAL - only used for IRC and XMPP protocols at the moment
@@ -1409,6 +1643,10 @@ enable=true
         [gateway.inout.options]
         webhookurl="https://discordapp.com/api/webhooks/123456789123456789/C9WPqExYWONPDZabcdef-def1434FGFjstasJX9pYht73y"
 
+    [[gateway.inout]]
+    account="zulip.streamchat"
+    channel="general/topic:mytopic"
+
     #API example
     #[[gateway.inout]]
     #account="api.local"

matterclient/channels.go

@@ -5,7 +5,6 @@ import (
 	"strings"
 
 	"github.com/mattermost/mattermost-server/model"
-	"github.com/sirupsen/logrus"
 )
 
 // GetChannels returns all channels we're members off
@@ -155,11 +154,11 @@ func (m *MMClient) JoinChannel(channelId string) error { //nolint:golint
 	defer m.RUnlock()
 	for _, c := range m.Team.Channels {
 		if c.Id == channelId {
-			m.log.Debug("Not joining ", channelId, " already joined.")
+			m.logger.Debug("Not joining ", channelId, " already joined.")
 			return nil
 		}
 	}
-	m.log.Debug("Joining ", channelId)
+	m.logger.Debug("Joining ", channelId)
 	_, resp := m.Client.AddChannelMember(channelId, m.User.Id)
 	if resp.Error != nil {
 		return resp.Error
@@ -189,19 +188,19 @@ func (m *MMClient) UpdateChannels() error {
 
 func (m *MMClient) UpdateChannelHeader(channelId string, header string) { //nolint:golint
 	channel := &model.Channel{Id: channelId, Header: header}
-	m.log.Debugf("updating channelheader %#v, %#v", channelId, header)
+	m.logger.Debugf("updating channelheader %#v, %#v", channelId, header)
 	_, resp := m.Client.UpdateChannel(channel)
 	if resp.Error != nil {
-		logrus.Error(resp.Error)
+		m.logger.Error(resp.Error)
 	}
 }
 
 func (m *MMClient) UpdateLastViewed(channelId string) error { //nolint:golint
-	m.log.Debugf("posting lastview %#v", channelId)
+	m.logger.Debugf("posting lastview %#v", channelId)
 	view := &model.ChannelView{ChannelId: channelId}
 	_, resp := m.Client.ViewChannel(m.User.Id, view)
 	if resp.Error != nil {
-		m.log.Errorf("ChannelView update for %s failed: %s", channelId, resp.Error)
+		m.logger.Errorf("ChannelView update for %s failed: %s", channelId, resp.Error)
 		return resp.Error
 	}
 	return nil

matterclient/helpers.go

@@ -22,7 +22,7 @@ func (m *MMClient) doLogin(firstConnection bool, b *backoff.Backoff) error {
 	var logmsg = "trying login"
 	var err error
 	for {
-		m.log.Debugf("%s %s %s %s", logmsg, m.Credentials.Team, m.Credentials.Login, m.Credentials.Server)
+		m.logger.Debugf("%s %s %s %s", logmsg, m.Credentials.Team, m.Credentials.Login, m.Credentials.Server)
 		if m.Credentials.Token != "" {
 			resp, err = m.doLoginToken()
 			if err != nil {
@@ -34,14 +34,14 @@ func (m *MMClient) doLogin(firstConnection bool, b *backoff.Backoff) error {
 		appErr = resp.Error
 		if appErr != nil {
 			d := b.Duration()
-			m.log.Debug(appErr.DetailedError)
+			m.logger.Debug(appErr.DetailedError)
 			if firstConnection {
 				if appErr.Message == "" {
 					return errors.New(appErr.DetailedError)
 				}
 				return errors.New(appErr.Message)
 			}
-			m.log.Debugf("LOGIN: %s, reconnecting in %s", appErr, d)
+			m.logger.Debugf("LOGIN: %s, reconnecting in %s", appErr, d)
 			time.Sleep(d)
 			logmsg = "retrying login"
 			continue
@@ -59,17 +59,17 @@ func (m *MMClient) doLoginToken() (*model.Response, error) {
 	m.Client.AuthType = model.HEADER_BEARER
 	m.Client.AuthToken = m.Credentials.Token
 	if m.Credentials.CookieToken {
-		m.log.Debugf(logmsg + " with cookie (MMAUTH) token")
+		m.logger.Debugf(logmsg + " with cookie (MMAUTH) token")
 		m.Client.HttpClient.Jar = m.createCookieJar(m.Credentials.Token)
 	} else {
-		m.log.Debugf(logmsg + " with personal token")
+		m.logger.Debugf(logmsg + " with personal token")
 	}
 	m.User, resp = m.Client.GetMe("")
 	if resp.Error != nil {
 		return resp, resp.Error
 	}
 	if m.User == nil {
-		m.log.Errorf("LOGIN TOKEN: %s is invalid", m.Credentials.Pass)
+		m.logger.Errorf("LOGIN TOKEN: %s is invalid", m.Credentials.Pass)
 		return resp, errors.New("invalid token")
 	}
 	return resp, nil
@@ -126,7 +126,7 @@ func (m *MMClient) initUser() error {
 	defer m.Unlock()
 	// we only load all team data on initial login.
 	// all other updates are for channels from our (primary) team only.
-	//m.log.Debug("initUser(): loading all team data")
+	//m.logger.Debug("initUser(): loading all team data")
 	teams, resp := m.Client.GetTeamsForUser(m.User.Id, "")
 	if resp.Error != nil {
 		return resp.Error
@@ -156,7 +156,7 @@ func (m *MMClient) initUser() error {
 		m.OtherTeams = append(m.OtherTeams, t)
 		if team.Name == m.Credentials.Team {
 			m.Team = t
-			m.log.Debugf("initUser(): found our team %s (id: %s)", team.Name, team.Id)
+			m.logger.Debugf("initUser(): found our team %s (id: %s)", team.Name, team.Id)
 		}
 		// add all users
 		for k, v := range t.Users {
@@ -180,10 +180,10 @@ func (m *MMClient) serverAlive(firstConnection bool, b *backoff.Backoff) error {
 		}
 		m.ServerVersion = resp.ServerVersion
 		if m.ServerVersion == "" {
-			m.log.Debugf("Server not up yet, reconnecting in %s", d)
+			m.logger.Debugf("Server not up yet, reconnecting in %s", d)
 			time.Sleep(d)
 		} else {
-			m.log.Infof("Found version %s", m.ServerVersion)
+			m.logger.Infof("Found version %s", m.ServerVersion)
 			return nil
 		}
 	}
@@ -207,7 +207,7 @@ func (m *MMClient) wsConnect() {
 	header := http.Header{}
 	header.Set(model.HEADER_AUTH, "BEARER "+m.Client.AuthToken)
 
-	m.log.Debugf("WsClient: making connection: %s", wsurl)
+	m.logger.Debugf("WsClient: making connection: %s", wsurl)
 	for {
 		wsDialer := &websocket.Dialer{
 			TLSClientConfig: &tls.Config{InsecureSkipVerify: m.SkipTLSVerify}, //nolint:gosec
@@ -217,14 +217,14 @@ func (m *MMClient) wsConnect() {
 		m.WsClient, _, err = wsDialer.Dial(wsurl, header)
 		if err != nil {
 			d := b.Duration()
-			m.log.Debugf("WSS: %s, reconnecting in %s", err, d)
+			m.logger.Debugf("WSS: %s, reconnecting in %s", err, d)
 			time.Sleep(d)
 			continue
 		}
 		break
 	}
 
-	m.log.Debug("WsClient: connected")
+	m.logger.Debug("WsClient: connected")
 	m.WsSequence = 1
 	m.WsPingChan = make(chan *model.WebSocketResponse)
 	// only start to parse WS messages when login is completely done
@@ -252,7 +252,7 @@ func (m *MMClient) checkAlive() error {
 	if resp.Error != nil {
 		return resp.Error
 	}
-	m.log.Debug("WS PING")
+	m.logger.Debug("WS PING")
 	return m.sendWSRequest("ping", nil)
 }
 
@@ -262,7 +262,7 @@ func (m *MMClient) sendWSRequest(action string, data map[string]interface{}) err
 	req.Action = action
 	req.Data = data
 	m.WsSequence++
-	m.log.Debugf("sendWsRequest %#v", req)
+	m.logger.Debugf("sendWsRequest %#v", req)
 	return m.WsClient.WriteJSON(req)
 }
 

matterclient/matterclient.go

@@ -8,7 +8,7 @@ import (
 	"time"
 
 	"github.com/gorilla/websocket"
-	"github.com/hashicorp/golang-lru"
+	lru "github.com/hashicorp/golang-lru"
 	"github.com/jpillora/backoff"
 	prefixed "github.com/matterbridge/logrus-prefixed-formatter"
 	"github.com/mattermost/mattermost-server/model"
@@ -49,13 +49,13 @@ type Team struct {
 type MMClient struct {
 	sync.RWMutex
 	*Credentials
+
 	Team          *Team
 	OtherTeams    []*Team
 	Client        *model.Client4
 	User          *model.User
 	Users         map[string]*model.User
 	MessageChan   chan *Message
-	log           *logrus.Entry
 	WsClient      *websocket.Conn
 	WsQuit        bool
 	WsAway        bool
@@ -64,31 +64,61 @@ type MMClient struct {
 	WsPingChan    chan *model.WebSocketResponse
 	ServerVersion string
 	OnWsConnect   func()
-	lruCache      *lru.Cache
+
+	logger     *logrus.Entry
+	rootLogger *logrus.Logger
+	lruCache   *lru.Cache
 }
 
-func New(login, pass, team, server string) *MMClient {
+// New will instantiate a new Matterclient with the specified login details without connecting.
-	cred := &Credentials{Login: login, Pass: pass, Team: team, Server: server}
+func New(login string, pass string, team string, server string) *MMClient {
-	mmclient := &MMClient{Credentials: cred, MessageChan: make(chan *Message, 100), Users: make(map[string]*model.User)}
+	rootLogger := logrus.New()
-	logrus.SetFormatter(&prefixed.TextFormatter{PrefixPadding: 13, DisableColors: true})
+	rootLogger.SetFormatter(&prefixed.TextFormatter{
-	mmclient.log = logrus.WithFields(logrus.Fields{"prefix": "matterclient"})
+		PrefixPadding: 13,
-	mmclient.lruCache, _ = lru.New(500)
+		DisableColors: true,
-	return mmclient
+	})
+
+	cred := &Credentials{
+		Login:  login,
+		Pass:   pass,
+		Team:   team,
+		Server: server,
+	}
+
+	cache, _ := lru.New(500)
+	return &MMClient{
+		Credentials: cred,
+		MessageChan: make(chan *Message, 100),
+		Users:       make(map[string]*model.User),
+		rootLogger:  rootLogger,
+		lruCache:    cache,
+		logger:      rootLogger.WithFields(logrus.Fields{"prefix": "matterclient"}),
+	}
 }
 
+// SetDebugLog activates debugging logging on all Matterclient log output.
 func (m *MMClient) SetDebugLog() {
-	logrus.SetFormatter(&prefixed.TextFormatter{PrefixPadding: 13, DisableColors: true, FullTimestamp: false, ForceFormatting: true})
+	m.rootLogger.SetFormatter(&prefixed.TextFormatter{
+		PrefixPadding:   13,
+		DisableColors:   true,
+		FullTimestamp:   false,
+		ForceFormatting: true,
+	})
 }
 
+// SetLogLevel tries to parse the specified level and if successful sets
+// the log level accordingly. Accepted levels are: 'debug', 'info', 'warn',
+// 'error', 'fatal' and 'panic'.
 func (m *MMClient) SetLogLevel(level string) {
 	l, err := logrus.ParseLevel(level)
 	if err != nil {
-		logrus.SetLevel(logrus.InfoLevel)
+		m.logger.Warnf("Failed to parse specified log-level '%s': %#v", level, err)
-		return
+	} else {
+		m.rootLogger.SetLevel(l)
 	}
-	logrus.SetLevel(l)
 }
 
+// Login tries to connect the client with the loging details with which it was initialized.
 func (m *MMClient) Login() error {
 	// check if this is a first connect or a reconnection
 	firstConnection := true
@@ -131,13 +161,14 @@ func (m *MMClient) Login() error {
 	return nil
 }
 
+// Logout disconnects the client from the chat server.
 func (m *MMClient) Logout() error {
-	m.log.Debugf("logout as %s (team: %s) on %s", m.Credentials.Login, m.Credentials.Team, m.Credentials.Server)
+	m.logger.Debugf("logout as %s (team: %s) on %s", m.Credentials.Login, m.Credentials.Team, m.Credentials.Server)
 	m.WsQuit = true
 	m.WsClient.Close()
 	m.WsClient.UnderlyingConn().Close()
 	if strings.Contains(m.Credentials.Pass, model.SESSION_COOKIE_TOKEN) {
-		m.log.Debug("Not invalidating session in logout, credential is a token")
+		m.logger.Debug("Not invalidating session in logout, credential is a token")
 		return nil
 	}
 	_, resp := m.Client.Logout()
@@ -147,13 +178,16 @@ func (m *MMClient) Logout() error {
 	return nil
 }
 
+// WsReceiver implements the core loop that manages the connection to the chat server. In
+// case of a disconnect it will try to reconnect. A call to this method is blocking until
+// the 'WsQuite' field of the MMClient object is set to 'true'.
 func (m *MMClient) WsReceiver() {
 	for {
 		var rawMsg json.RawMessage
 		var err error
 
 		if m.WsQuit {
-			m.log.Debug("exiting WsReceiver")
+			m.logger.Debug("exiting WsReceiver")
 			return
 		}
 
@@ -163,14 +197,14 @@ func (m *MMClient) WsReceiver() {
 		}
 
 		if _, rawMsg, err = m.WsClient.ReadMessage(); err != nil {
-			m.log.Error("error:", err)
+			m.logger.Error("error:", err)
 			// reconnect
 			m.wsConnect()
 		}
 
 		var event model.WebSocketEvent
 		if err := json.Unmarshal(rawMsg, &event); err == nil && event.IsValid() {
-			m.log.Debugf("WsReceiver event: %#v", event)
+			m.logger.Debugf("WsReceiver event: %#v", event)
 			msg := &Message{Raw: &event, Team: m.Credentials.Team}
 			m.parseMessage(msg)
 			// check if we didn't empty the message
@@ -189,40 +223,42 @@ func (m *MMClient) WsReceiver() {
 
 		var response model.WebSocketResponse
 		if err := json.Unmarshal(rawMsg, &response); err == nil && response.IsValid() {
-			m.log.Debugf("WsReceiver response: %#v", response)
+			m.logger.Debugf("WsReceiver response: %#v", response)
 			m.parseResponse(response)
-			continue
 		}
 	}
 }
 
+// StatusLoop implements a ping-cycle that ensures that the connection to the chat servers
+// remains alive. In case of a disconnect it will try to reconnect. A call to this method
+// is blocking until the 'WsQuite' field of the MMClient object is set to 'true'.
 func (m *MMClient) StatusLoop() {
 	retries := 0
 	backoff := time.Second * 60
 	if m.OnWsConnect != nil {
 		m.OnWsConnect()
 	}
-	m.log.Debug("StatusLoop:", m.OnWsConnect != nil)
+	m.logger.Debug("StatusLoop:", m.OnWsConnect != nil)
 	for {
 		if m.WsQuit {
 			return
 		}
 		if m.WsConnected {
 			if err := m.checkAlive(); err != nil {
-				logrus.Errorf("Connection is not alive: %#v", err)
+				m.logger.Errorf("Connection is not alive: %#v", err)
 			}
 			select {
 			case <-m.WsPingChan:
-				m.log.Debug("WS PONG received")
+				m.logger.Debug("WS PONG received")
 				backoff = time.Second * 60
 			case <-time.After(time.Second * 5):
 				if retries > 3 {
-					m.log.Debug("StatusLoop() timeout")
+					m.logger.Debug("StatusLoop() timeout")
 					m.Logout()
 					m.WsQuit = false
 					err := m.Login()
 					if err != nil {
-						logrus.Errorf("Login failed: %#v", err)
+						m.logger.Errorf("Login failed: %#v", err)
 						break
 					}
 					if m.OnWsConnect != nil {

matterclient/messages.go

@@ -10,14 +10,14 @@ func (m *MMClient) parseActionPost(rmsg *Message) {
 	// add post to cache, if it already exists don't relay this again.
 	// this should fix reposts
 	if ok, _ := m.lruCache.ContainsOrAdd(digestString(rmsg.Raw.Data["post"].(string)), true); ok {
-		m.log.Debugf("message %#v in cache, not processing again", rmsg.Raw.Data["post"].(string))
+		m.logger.Debugf("message %#v in cache, not processing again", rmsg.Raw.Data["post"].(string))
 		rmsg.Text = ""
 		return
 	}
 	data := model.PostFromJson(strings.NewReader(rmsg.Raw.Data["post"].(string)))
 	// we don't have the user, refresh the userlist
 	if m.GetUser(data.UserId) == nil {
-		m.log.Infof("User '%v' is not known, ignoring message '%#v'",
+		m.logger.Infof("User '%v' is not known, ignoring message '%#v'",
 			data.UserId, data)
 		return
 	}
@@ -54,7 +54,7 @@ func (m *MMClient) parseMessage(rmsg *Message) {
 		}
 	case "group_added":
 		if err := m.UpdateChannels(); err != nil {
-			m.log.Errorf("failed to update channels: %#v", err)
+			m.logger.Errorf("failed to update channels: %#v", err)
 		}
 		/*
 			case model.ACTION_USER_REMOVED:
@@ -178,18 +178,18 @@ func (m *MMClient) SendDirectMessage(toUserId string, msg string, rootId string)
 }
 
 func (m *MMClient) SendDirectMessageProps(toUserId string, msg string, rootId string, props map[string]interface{}) { //nolint:golint
-	m.log.Debugf("SendDirectMessage to %s, msg %s", toUserId, msg)
+	m.logger.Debugf("SendDirectMessage to %s, msg %s", toUserId, msg)
 	// create DM channel (only happens on first message)
 	_, resp := m.Client.CreateDirectChannel(m.User.Id, toUserId)
 	if resp.Error != nil {
-		m.log.Debugf("SendDirectMessage to %#v failed: %s", toUserId, resp.Error)
+		m.logger.Debugf("SendDirectMessage to %#v failed: %s", toUserId, resp.Error)
 		return
 	}
 	channelName := model.GetDMNameFromIds(toUserId, m.User.Id)
 
 	// update our channels
 	if err := m.UpdateChannels(); err != nil {
-		m.log.Errorf("failed to update channels: %#v", err)
+		m.logger.Errorf("failed to update channels: %#v", err)
 	}
 
 	// build & send the message

matterclient/users.go

@@ -124,7 +124,7 @@ func (m *MMClient) UpdateUserNick(nick string) error {
 func (m *MMClient) UsernamesInChannel(channelId string) []string { //nolint:golint
 	res, resp := m.Client.GetChannelMembers(channelId, 0, 50000, "")
 	if resp.Error != nil {
-		m.log.Errorf("UsernamesInChannel(%s) failed: %s", channelId, resp.Error)
+		m.logger.Errorf("UsernamesInChannel(%s) failed: %s", channelId, resp.Error)
 		return []string{}
 	}
 	allusers := m.GetUsers()

vendor/github.com/Baozisoftware/qrcode-terminal-go/.gitignore

@@ -0,0 +1,24 @@
+# Compiled Object files, Static and Dynamic libs (Shared Objects)
+*.o
+*.a
+*.so
+
+# Folders
+_obj
+_test
+
+# Architecture specific extensions/prefixes
+*.[568vq]
+[568vq].out
+
+*.cgo1.go
+*.cgo2.c
+_cgo_defun.c
+_cgo_gotypes.go
+_cgo_export.*
+
+_testmain.go
+
+*.exe
+*.test
+*.prof

vendor/github.com/Baozisoftware/qrcode-terminal-go/LICENSE

@@ -0,0 +1,29 @@
+BSD 3-Clause License
+
+Copyright (c) 2017, Baozisoftware
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+* Neither the name of the copyright holder nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

vendor/github.com/Baozisoftware/qrcode-terminal-go/README.md

@@ -0,0 +1,39 @@
+
+# qrcode-terminal-go
+QRCode terminal for golang.
+
+# Example
+```go
+package main
+
+import "github.com/Baozisoftware/qrcode-terminal-go"
+
+func main() {
+	Test1()
+	Test2()
+}
+
+func Test1(){
+	content := "Hello, 世界"
+	obj := qrcodeTerminal.New()
+	obj.Get(content).Print()
+}
+
+func Test2(){
+	content := "https://github.com/Baozisoftware/qrcode-terminal-go"
+	obj := qrcodeTerminal.New2(qrcodeTerminal.ConsoleColors.BrightBlue,qrcodeTerminal.ConsoleColors.BrightGreen,qrcodeTerminal.QRCodeRecoveryLevels.Low)
+	obj.Get([]byte(content)).Print()
+}
+```
+
+## Screenshots
+### Windows XP
+![winxp](https://github.com/Baozisoftware/qrcode-terminal-go/blob/master/screenshots/winxp.png)
+### Windows 7
+![win7](https://github.com/Baozisoftware/qrcode-terminal-go/blob/master/screenshots/win7.png)
+### Windows 10
+![win10](https://github.com/Baozisoftware/qrcode-terminal-go/blob/master/screenshots/win10.png)
+### Ubuntu
+![ubuntu](https://github.com/Baozisoftware/qrcode-terminal-go/blob/master/screenshots/ubuntu.png)
+### macOS
+![macos](https://github.com/Baozisoftware/qrcode-terminal-go/blob/master/screenshots/macos.png)

vendor/github.com/Baozisoftware/qrcode-terminal-go/qrcodeTerminal.go

@@ -0,0 +1,155 @@
+package qrcodeTerminal
+
+import (
+	"fmt"
+
+	"github.com/skip2/go-qrcode"
+	"github.com/mattn/go-colorable"
+	"image/png"
+	nbytes "bytes"
+)
+
+type consoleColor string
+type consoleColors struct {
+	NormalBlack   consoleColor
+	NormalRed     consoleColor
+	NormalGreen   consoleColor
+	NormalYellow  consoleColor
+	NormalBlue    consoleColor
+	NormalMagenta consoleColor
+	NormalCyan    consoleColor
+	NormalWhite   consoleColor
+	BrightBlack   consoleColor
+	BrightRed     consoleColor
+	BrightGreen   consoleColor
+	BrightYellow  consoleColor
+	BrightBlue    consoleColor
+	BrightMagenta consoleColor
+	BrightCyan    consoleColor
+	BrightWhite   consoleColor
+}
+type qrcodeRecoveryLevel qrcode.RecoveryLevel
+type qrcodeRecoveryLevels struct {
+	Low     qrcodeRecoveryLevel
+	Medium  qrcodeRecoveryLevel
+	High    qrcodeRecoveryLevel
+	Highest qrcodeRecoveryLevel
+}
+
+var (
+	ConsoleColors consoleColors = consoleColors{
+		NormalBlack:   "\033[38;5;0m  \033[0m",
+		NormalRed:     "\033[38;5;1m  \033[0m",
+		NormalGreen:   "\033[38;5;2m  \033[0m",
+		NormalYellow:  "\033[38;5;3m  \033[0m",
+		NormalBlue:    "\033[38;5;4m  \033[0m",
+		NormalMagenta: "\033[38;5;5m  \033[0m",
+		NormalCyan:    "\033[38;5;6m  \033[0m",
+		NormalWhite:   "\033[38;5;7m  \033[0m",
+		BrightBlack:   "\033[48;5;0m  \033[0m",
+		BrightRed:     "\033[48;5;1m  \033[0m",
+		BrightGreen:   "\033[48;5;2m  \033[0m",
+		BrightYellow:  "\033[48;5;3m  \033[0m",
+		BrightBlue:    "\033[48;5;4m  \033[0m",
+		BrightMagenta: "\033[48;5;5m  \033[0m",
+		BrightCyan:    "\033[48;5;6m  \033[0m",
+		BrightWhite:   "\033[48;5;7m  \033[0m"}
+	QRCodeRecoveryLevels = qrcodeRecoveryLevels{
+		Low:     qrcodeRecoveryLevel(qrcode.Low),
+		Medium:  qrcodeRecoveryLevel(qrcode.Medium),
+		High:    qrcodeRecoveryLevel(qrcode.High),
+		Highest: qrcodeRecoveryLevel(qrcode.Highest)}
+)
+
+type QRCodeString string
+
+func (v *QRCodeString) Print() {
+	fmt.Fprint(outer, *v)
+}
+
+type qrcodeTerminal struct {
+	front consoleColor
+	back  consoleColor
+	level qrcodeRecoveryLevel
+}
+
+func (v *qrcodeTerminal) Get(content interface{}) (result *QRCodeString) {
+	var qr *qrcode.QRCode
+	var err error
+	if t, ok := content.(string); ok {
+		qr, err = qrcode.New(t, qrcode.RecoveryLevel(v.level))
+	} else if t, ok := content.([]byte); ok {
+		qr, err = qrcode.New(string(t), qrcode.RecoveryLevel(v.level))
+	}
+	if qr != nil && err == nil {
+		data := qr.Bitmap()
+		result = v.getQRCodeString(data)
+	}
+	return
+}
+
+func (v *qrcodeTerminal) Get2(bytes []byte) (result *QRCodeString) {
+	data, err := parseQR(bytes)
+	if err == nil {
+		result = v.getQRCodeString(data)
+	}
+	return
+}
+
+func New2(front, back consoleColor, level qrcodeRecoveryLevel) *qrcodeTerminal {
+	obj := qrcodeTerminal{front: front, back: back, level: level}
+	return &obj
+}
+
+func New() *qrcodeTerminal {
+	front, back, level := ConsoleColors.BrightBlack, ConsoleColors.BrightWhite, QRCodeRecoveryLevels.Medium
+	return New2(front, back, level)
+}
+
+func (v *qrcodeTerminal) getQRCodeString(data [][]bool) (result *QRCodeString) {
+	str := ""
+	for ir, row := range data {
+		lr := len(row)
+		if ir == 0 || ir == 1 || ir == 2 ||
+			ir == lr-1 || ir == lr-2 || ir == lr-3 {
+			continue
+		}
+		for ic, col := range row {
+			lc := len(data)
+			if ic == 0 || ic == 1 || ic == 2 ||
+				ic == lc-1 || ic == lc-2 || ic == lc-3 {
+				continue
+			}
+			if col {
+				str += fmt.Sprint(v.front)
+			} else {
+				str += fmt.Sprint(v.back)
+			}
+		}
+		str += fmt.Sprintln()
+	}
+	obj := QRCodeString(str)
+	result = &obj
+	return
+}
+
+func parseQR(bytes []byte) (data [][]bool, err error) {
+	r := nbytes.NewReader(bytes)
+	img, err := png.Decode(r)
+	if err == nil {
+		rect := img.Bounds()
+		mx, my := rect.Max.X, rect.Max.Y
+		data = make([][]bool, mx)
+		for x := 0; x < mx; x++ {
+			data[x] = make([]bool, my)
+			for y := 0; y < my; y++ {
+				c := img.At(x, y)
+				r, _, _, _ := c.RGBA()
+				data[x][y] = r == 0
+			}
+		}
+	}
+	return
+}
+
+var outer = colorable.NewColorableStdout()

vendor/github.com/Jeffail/gabs/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2014 Ashley Jeffs
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

vendor/github.com/Jeffail/gabs/README.md

@@ -0,0 +1,315 @@
+![Gabs](gabs_logo.png "Gabs")
+
+Gabs is a small utility for dealing with dynamic or unknown JSON structures in
+golang. It's pretty much just a helpful wrapper around the golang
+`json.Marshal/json.Unmarshal` behaviour and `map[string]interface{}` objects.
+It does nothing spectacular except for being fabulous.
+
+https://godoc.org/github.com/Jeffail/gabs
+
+## How to install:
+
+``` bash
+go get github.com/Jeffail/gabs
+```
+
+## How to use
+
+### Parsing and searching JSON
+
+``` go
+...
+
+import "github.com/Jeffail/gabs"
+
+jsonParsed, err := gabs.ParseJSON([]byte(`{
+	"outter":{
+		"inner":{
+			"value1":10,
+			"value2":22
+		},
+		"alsoInner":{
+			"value1":20
+		}
+	}
+}`))
+
+var value float64
+var ok bool
+
+value, ok = jsonParsed.Path("outter.inner.value1").Data().(float64)
+// value == 10.0, ok == true
+
+value, ok = jsonParsed.Search("outter", "inner", "value1").Data().(float64)
+// value == 10.0, ok == true
+
+value, ok = jsonParsed.Path("does.not.exist").Data().(float64)
+// value == 0.0, ok == false
+
+exists := jsonParsed.Exists("outter", "inner", "value1")
+// exists == true
+
+exists := jsonParsed.Exists("does", "not", "exist")
+// exists == false
+
+exists := jsonParsed.ExistsP("does.not.exist")
+// exists == false
+
+...
+```
+
+### Iterating objects
+
+``` go
+...
+
+jsonParsed, _ := gabs.ParseJSON([]byte(`{"object":{ "first": 1, "second": 2, "third": 3 }}`))
+
+// S is shorthand for Search
+children, _ := jsonParsed.S("object").ChildrenMap()
+for key, child := range children {
+	fmt.Printf("key: %v, value: %v\n", key, child.Data().(string))
+}
+
+...
+```
+
+### Iterating arrays
+
+``` go
+...
+
+jsonParsed, _ := gabs.ParseJSON([]byte(`{"array":[ "first", "second", "third" ]}`))
+
+// S is shorthand for Search
+children, _ := jsonParsed.S("array").Children()
+for _, child := range children {
+	fmt.Println(child.Data().(string))
+}
+
+...
+```
+
+Will print:
+
+```
+first
+second
+third
+```
+
+Children() will return all children of an array in order. This also works on
+objects, however, the children will be returned in a random order.
+
+### Searching through arrays
+
+If your JSON structure contains arrays you can still search the fields of the
+objects within the array, this returns a JSON array containing the results for
+each element.
+
+``` go
+...
+
+jsonParsed, _ := gabs.ParseJSON([]byte(`{"array":[ {"value":1}, {"value":2}, {"value":3} ]}`))
+fmt.Println(jsonParsed.Path("array.value").String())
+
+...
+```
+
+Will print:
+
+```
+[1,2,3]
+```
+
+### Generating JSON
+
+``` go
+...
+
+jsonObj := gabs.New()
+// or gabs.Consume(jsonObject) to work on an existing map[string]interface{}
+
+jsonObj.Set(10, "outter", "inner", "value")
+jsonObj.SetP(20, "outter.inner.value2")
+jsonObj.Set(30, "outter", "inner2", "value3")
+
+fmt.Println(jsonObj.String())
+
+...
+```
+
+Will print:
+
+```
+{"outter":{"inner":{"value":10,"value2":20},"inner2":{"value3":30}}}
+```
+
+To pretty-print:
+
+``` go
+...
+
+fmt.Println(jsonObj.StringIndent("", "  "))
+
+...
+```
+
+Will print:
+
+```
+{
+  "outter": {
+    "inner": {
+      "value": 10,
+      "value2": 20
+    },
+    "inner2": {
+      "value3": 30
+    }
+  }
+}
+```
+
+### Generating Arrays
+
+``` go
+...
+
+jsonObj := gabs.New()
+
+jsonObj.Array("foo", "array")
+// Or .ArrayP("foo.array")
+
+jsonObj.ArrayAppend(10, "foo", "array")
+jsonObj.ArrayAppend(20, "foo", "array")
+jsonObj.ArrayAppend(30, "foo", "array")
+
+fmt.Println(jsonObj.String())
+
+...
+```
+
+Will print:
+
+```
+{"foo":{"array":[10,20,30]}}
+```
+
+Working with arrays by index:
+
+``` go
+...
+
+jsonObj := gabs.New()
+
+// Create an array with the length of 3
+jsonObj.ArrayOfSize(3, "foo")
+
+jsonObj.S("foo").SetIndex("test1", 0)
+jsonObj.S("foo").SetIndex("test2", 1)
+
+// Create an embedded array with the length of 3
+jsonObj.S("foo").ArrayOfSizeI(3, 2)
+
+jsonObj.S("foo").Index(2).SetIndex(1, 0)
+jsonObj.S("foo").Index(2).SetIndex(2, 1)
+jsonObj.S("foo").Index(2).SetIndex(3, 2)
+
+fmt.Println(jsonObj.String())
+
+...
+```
+
+Will print:
+
+```
+{"foo":["test1","test2",[1,2,3]]}
+```
+
+### Converting back to JSON
+
+This is the easiest part:
+
+``` go
+...
+
+jsonParsedObj, _ := gabs.ParseJSON([]byte(`{
+	"outter":{
+		"values":{
+			"first":10,
+			"second":11
+		}
+	},
+	"outter2":"hello world"
+}`))
+
+jsonOutput := jsonParsedObj.String()
+// Becomes `{"outter":{"values":{"first":10,"second":11}},"outter2":"hello world"}`
+
+...
+```
+
+And to serialize a specific segment is as simple as:
+
+``` go
+...
+
+jsonParsedObj := gabs.ParseJSON([]byte(`{
+	"outter":{
+		"values":{
+			"first":10,
+			"second":11
+		}
+	},
+	"outter2":"hello world"
+}`))
+
+jsonOutput := jsonParsedObj.Search("outter").String()
+// Becomes `{"values":{"first":10,"second":11}}`
+
+...
+```
+
+### Merge two containers
+
+You can merge a JSON structure into an existing one, where collisions will be
+converted into a JSON array.
+
+``` go
+jsonParsed1, _ := ParseJSON([]byte(`{"outter": {"value1": "one"}}`))
+jsonParsed2, _ := ParseJSON([]byte(`{"outter": {"inner": {"value3": "three"}}, "outter2": {"value2": "two"}}`))
+
+jsonParsed1.Merge(jsonParsed2)
+// Becomes `{"outter":{"inner":{"value3":"three"},"value1":"one"},"outter2":{"value2":"two"}}`
+```
+
+Arrays are merged:
+
+``` go
+jsonParsed1, _ := ParseJSON([]byte(`{"array": ["one"]}`))
+jsonParsed2, _ := ParseJSON([]byte(`{"array": ["two"]}`))
+
+jsonParsed1.Merge(jsonParsed2)
+// Becomes `{"array":["one", "two"]}`
+```
+
+### Parsing Numbers
+
+Gabs uses the `json` package under the bonnet, which by default will parse all
+number values into `float64`. If you need to parse `Int` values then you should
+use a `json.Decoder` (https://golang.org/pkg/encoding/json/#Decoder):
+
+``` go
+sample := []byte(`{"test":{"int":10, "float":6.66}}`)
+dec := json.NewDecoder(bytes.NewReader(sample))
+dec.UseNumber()
+
+val, err := gabs.ParseJSONDecoder(dec)
+if err != nil {
+    t.Errorf("Failed to parse: %v", err)
+    return
+}
+
+intValue, err := val.Path("test.int").Data().(json.Number).Int64()
+```

vendor/github.com/Jeffail/gabs/gabs.go

@@ -0,0 +1,581 @@
+/*
+Copyright (c) 2014 Ashley Jeffs
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+// Package gabs implements a simplified wrapper around creating and parsing JSON.
+package gabs
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"io"
+	"io/ioutil"
+	"strings"
+)
+
+//--------------------------------------------------------------------------------------------------
+
+var (
+	// ErrOutOfBounds - Index out of bounds.
+	ErrOutOfBounds = errors.New("out of bounds")
+
+	// ErrNotObjOrArray - The target is not an object or array type.
+	ErrNotObjOrArray = errors.New("not an object or array")
+
+	// ErrNotObj - The target is not an object type.
+	ErrNotObj = errors.New("not an object")
+
+	// ErrNotArray - The target is not an array type.
+	ErrNotArray = errors.New("not an array")
+
+	// ErrPathCollision - Creating a path failed because an element collided with an existing value.
+	ErrPathCollision = errors.New("encountered value collision whilst building path")
+
+	// ErrInvalidInputObj - The input value was not a map[string]interface{}.
+	ErrInvalidInputObj = errors.New("invalid input object")
+
+	// ErrInvalidInputText - The input data could not be parsed.
+	ErrInvalidInputText = errors.New("input text could not be parsed")
+
+	// ErrInvalidPath - The filepath was not valid.
+	ErrInvalidPath = errors.New("invalid file path")
+
+	// ErrInvalidBuffer - The input buffer contained an invalid JSON string
+	ErrInvalidBuffer = errors.New("input buffer contained invalid JSON")
+)
+
+//--------------------------------------------------------------------------------------------------
+
+// Container - an internal structure that holds a reference to the core interface map of the parsed
+// json. Use this container to move context.
+type Container struct {
+	object interface{}
+}
+
+// Data - Return the contained data as an interface{}.
+func (g *Container) Data() interface{} {
+	if g == nil {
+		return nil
+	}
+	return g.object
+}
+
+//--------------------------------------------------------------------------------------------------
+
+// Path - Search for a value using dot notation.
+func (g *Container) Path(path string) *Container {
+	return g.Search(strings.Split(path, ".")...)
+}
+
+// Search - Attempt to find and return an object within the JSON structure by specifying the
+// hierarchy of field names to locate the target. If the search encounters an array and has not
+// reached the end target then it will iterate each object of the array for the target and return
+// all of the results in a JSON array.
+func (g *Container) Search(hierarchy ...string) *Container {
+	var object interface{}
+
+	object = g.Data()
+	for target := 0; target < len(hierarchy); target++ {
+		if mmap, ok := object.(map[string]interface{}); ok {
+			object, ok = mmap[hierarchy[target]]
+			if !ok {
+				return nil
+			}
+		} else if marray, ok := object.([]interface{}); ok {
+			tmpArray := []interface{}{}
+			for _, val := range marray {
+				tmpGabs := &Container{val}
+				res := tmpGabs.Search(hierarchy[target:]...)
+				if res != nil {
+					tmpArray = append(tmpArray, res.Data())
+				}
+			}
+			if len(tmpArray) == 0 {
+				return nil
+			}
+			return &Container{tmpArray}
+		} else {
+			return nil
+		}
+	}
+	return &Container{object}
+}
+
+// S - Shorthand method, does the same thing as Search.
+func (g *Container) S(hierarchy ...string) *Container {
+	return g.Search(hierarchy...)
+}
+
+// Exists - Checks whether a path exists.
+func (g *Container) Exists(hierarchy ...string) bool {
+	return g.Search(hierarchy...) != nil
+}
+
+// ExistsP - Checks whether a dot notation path exists.
+func (g *Container) ExistsP(path string) bool {
+	return g.Exists(strings.Split(path, ".")...)
+}
+
+// Index - Attempt to find and return an object within a JSON array by index.
+func (g *Container) Index(index int) *Container {
+	if array, ok := g.Data().([]interface{}); ok {
+		if index >= len(array) {
+			return &Container{nil}
+		}
+		return &Container{array[index]}
+	}
+	return &Container{nil}
+}
+
+// Children - Return a slice of all the children of the array. This also works for objects, however,
+// the children returned for an object will NOT be in order and you lose the names of the returned
+// objects this way.
+func (g *Container) Children() ([]*Container, error) {
+	if array, ok := g.Data().([]interface{}); ok {
+		children := make([]*Container, len(array))
+		for i := 0; i < len(array); i++ {
+			children[i] = &Container{array[i]}
+		}
+		return children, nil
+	}
+	if mmap, ok := g.Data().(map[string]interface{}); ok {
+		children := []*Container{}
+		for _, obj := range mmap {
+			children = append(children, &Container{obj})
+		}
+		return children, nil
+	}
+	return nil, ErrNotObjOrArray
+}
+
+// ChildrenMap - Return a map of all the children of an object.
+func (g *Container) ChildrenMap() (map[string]*Container, error) {
+	if mmap, ok := g.Data().(map[string]interface{}); ok {
+		children := map[string]*Container{}
+		for name, obj := range mmap {
+			children[name] = &Container{obj}
+		}
+		return children, nil
+	}
+	return nil, ErrNotObj
+}
+
+//--------------------------------------------------------------------------------------------------
+
+// Set - Set the value of a field at a JSON path, any parts of the path that do not exist will be
+// constructed, and if a collision occurs with a non object type whilst iterating the path an error
+// is returned.
+func (g *Container) Set(value interface{}, path ...string) (*Container, error) {
+	if len(path) == 0 {
+		g.object = value
+		return g, nil
+	}
+	var object interface{}
+	if g.object == nil {
+		g.object = map[string]interface{}{}
+	}
+	object = g.object
+	for target := 0; target < len(path); target++ {
+		if mmap, ok := object.(map[string]interface{}); ok {
+			if target == len(path)-1 {
+				mmap[path[target]] = value
+			} else if mmap[path[target]] == nil {
+				mmap[path[target]] = map[string]interface{}{}
+			}
+			object = mmap[path[target]]
+		} else {
+			return &Container{nil}, ErrPathCollision
+		}
+	}
+	return &Container{object}, nil
+}
+
+// SetP - Does the same as Set, but using a dot notation JSON path.
+func (g *Container) SetP(value interface{}, path string) (*Container, error) {
+	return g.Set(value, strings.Split(path, ".")...)
+}
+
+// SetIndex - Set a value of an array element based on the index.
+func (g *Container) SetIndex(value interface{}, index int) (*Container, error) {
+	if array, ok := g.Data().([]interface{}); ok {
+		if index >= len(array) {
+			return &Container{nil}, ErrOutOfBounds
+		}
+		array[index] = value
+		return &Container{array[index]}, nil
+	}
+	return &Container{nil}, ErrNotArray
+}
+
+// Object - Create a new JSON object at a path. Returns an error if the path contains a collision
+// with a non object type.
+func (g *Container) Object(path ...string) (*Container, error) {
+	return g.Set(map[string]interface{}{}, path...)
+}
+
+// ObjectP - Does the same as Object, but using a dot notation JSON path.
+func (g *Container) ObjectP(path string) (*Container, error) {
+	return g.Object(strings.Split(path, ".")...)
+}
+
+// ObjectI - Create a new JSON object at an array index. Returns an error if the object is not an
+// array or the index is out of bounds.
+func (g *Container) ObjectI(index int) (*Container, error) {
+	return g.SetIndex(map[string]interface{}{}, index)
+}
+
+// Array - Create a new JSON array at a path. Returns an error if the path contains a collision with
+// a non object type.
+func (g *Container) Array(path ...string) (*Container, error) {
+	return g.Set([]interface{}{}, path...)
+}
+
+// ArrayP - Does the same as Array, but using a dot notation JSON path.
+func (g *Container) ArrayP(path string) (*Container, error) {
+	return g.Array(strings.Split(path, ".")...)
+}
+
+// ArrayI - Create a new JSON array at an array index. Returns an error if the object is not an
+// array or the index is out of bounds.
+func (g *Container) ArrayI(index int) (*Container, error) {
+	return g.SetIndex([]interface{}{}, index)
+}
+
+// ArrayOfSize - Create a new JSON array of a particular size at a path. Returns an error if the
+// path contains a collision with a non object type.
+func (g *Container) ArrayOfSize(size int, path ...string) (*Container, error) {
+	a := make([]interface{}, size)
+	return g.Set(a, path...)
+}
+
+// ArrayOfSizeP - Does the same as ArrayOfSize, but using a dot notation JSON path.
+func (g *Container) ArrayOfSizeP(size int, path string) (*Container, error) {
+	return g.ArrayOfSize(size, strings.Split(path, ".")...)
+}
+
+// ArrayOfSizeI - Create a new JSON array of a particular size at an array index. Returns an error
+// if the object is not an array or the index is out of bounds.
+func (g *Container) ArrayOfSizeI(size, index int) (*Container, error) {
+	a := make([]interface{}, size)
+	return g.SetIndex(a, index)
+}
+
+// Delete - Delete an element at a JSON path, an error is returned if the element does not exist.
+func (g *Container) Delete(path ...string) error {
+	var object interface{}
+
+	if g.object == nil {
+		return ErrNotObj
+	}
+	object = g.object
+	for target := 0; target < len(path); target++ {
+		if mmap, ok := object.(map[string]interface{}); ok {
+			if target == len(path)-1 {
+				if _, ok := mmap[path[target]]; ok {
+					delete(mmap, path[target])
+				} else {
+					return ErrNotObj
+				}
+			}
+			object = mmap[path[target]]
+		} else {
+			return ErrNotObj
+		}
+	}
+	return nil
+}
+
+// DeleteP - Does the same as Delete, but using a dot notation JSON path.
+func (g *Container) DeleteP(path string) error {
+	return g.Delete(strings.Split(path, ".")...)
+}
+
+// Merge - Merges two gabs-containers
+func (g *Container) Merge(toMerge *Container) error {
+	var recursiveFnc func(map[string]interface{}, []string) error
+	recursiveFnc = func(mmap map[string]interface{}, path []string) error {
+		for key, value := range mmap {
+			newPath := append(path, key)
+			if g.Exists(newPath...) {
+				target := g.Search(newPath...)
+				switch t := value.(type) {
+				case map[string]interface{}:
+					switch targetV := target.Data().(type) {
+					case map[string]interface{}:
+						if err := recursiveFnc(t, newPath); err != nil {
+							return err
+						}
+					case []interface{}:
+						g.Set(append(targetV, t), newPath...)
+					default:
+						newSlice := append([]interface{}{}, targetV)
+						g.Set(append(newSlice, t), newPath...)
+					}
+				case []interface{}:
+					for _, valueOfSlice := range t {
+						if err := g.ArrayAppend(valueOfSlice, newPath...); err != nil {
+							return err
+						}
+					}
+				default:
+					switch targetV := target.Data().(type) {
+					case []interface{}:
+						g.Set(append(targetV, t), newPath...)
+					default:
+						newSlice := append([]interface{}{}, targetV)
+						g.Set(append(newSlice, t), newPath...)
+					}
+				}
+			} else {
+				// path doesn't exist. So set the value
+				if _, err := g.Set(value, newPath...); err != nil {
+					return err
+				}
+			}
+		}
+		return nil
+	}
+	if mmap, ok := toMerge.Data().(map[string]interface{}); ok {
+		return recursiveFnc(mmap, []string{})
+	}
+	return nil
+}
+
+//--------------------------------------------------------------------------------------------------
+
+/*
+Array modification/search - Keeping these options simple right now, no need for anything more
+complicated since you can just cast to []interface{}, modify and then reassign with Set.
+*/
+
+// ArrayAppend - Append a value onto a JSON array. If the target is not a JSON array then it will be
+// converted into one, with its contents as the first element of the array.
+func (g *Container) ArrayAppend(value interface{}, path ...string) error {
+	if array, ok := g.Search(path...).Data().([]interface{}); ok {
+		array = append(array, value)
+		_, err := g.Set(array, path...)
+		return err
+	}
+
+	newArray := []interface{}{}
+	if d := g.Search(path...).Data(); d != nil {
+		newArray = append(newArray, d)
+	}
+	newArray = append(newArray, value)
+
+	_, err := g.Set(newArray, path...)
+	return err
+}
+
+// ArrayAppendP - Append a value onto a JSON array using a dot notation JSON path.
+func (g *Container) ArrayAppendP(value interface{}, path string) error {
+	return g.ArrayAppend(value, strings.Split(path, ".")...)
+}
+
+// ArrayRemove - Remove an element from a JSON array.
+func (g *Container) ArrayRemove(index int, path ...string) error {
+	if index < 0 {
+		return ErrOutOfBounds
+	}
+	array, ok := g.Search(path...).Data().([]interface{})
+	if !ok {
+		return ErrNotArray
+	}
+	if index < len(array) {
+		array = append(array[:index], array[index+1:]...)
+	} else {
+		return ErrOutOfBounds
+	}
+	_, err := g.Set(array, path...)
+	return err
+}
+
+// ArrayRemoveP - Remove an element from a JSON array using a dot notation JSON path.
+func (g *Container) ArrayRemoveP(index int, path string) error {
+	return g.ArrayRemove(index, strings.Split(path, ".")...)
+}
+
+// ArrayElement - Access an element from a JSON array.
+func (g *Container) ArrayElement(index int, path ...string) (*Container, error) {
+	if index < 0 {
+		return &Container{nil}, ErrOutOfBounds
+	}
+	array, ok := g.Search(path...).Data().([]interface{})
+	if !ok {
+		return &Container{nil}, ErrNotArray
+	}
+	if index < len(array) {
+		return &Container{array[index]}, nil
+	}
+	return &Container{nil}, ErrOutOfBounds
+}
+
+// ArrayElementP - Access an element from a JSON array using a dot notation JSON path.
+func (g *Container) ArrayElementP(index int, path string) (*Container, error) {
+	return g.ArrayElement(index, strings.Split(path, ".")...)
+}
+
+// ArrayCount - Count the number of elements in a JSON array.
+func (g *Container) ArrayCount(path ...string) (int, error) {
+	if array, ok := g.Search(path...).Data().([]interface{}); ok {
+		return len(array), nil
+	}
+	return 0, ErrNotArray
+}
+
+// ArrayCountP - Count the number of elements in a JSON array using a dot notation JSON path.
+func (g *Container) ArrayCountP(path string) (int, error) {
+	return g.ArrayCount(strings.Split(path, ".")...)
+}
+
+//--------------------------------------------------------------------------------------------------
+
+// Bytes - Converts the contained object back to a JSON []byte blob.
+func (g *Container) Bytes() []byte {
+	if g.Data() != nil {
+		if bytes, err := json.Marshal(g.object); err == nil {
+			return bytes
+		}
+	}
+	return []byte("{}")
+}
+
+// BytesIndent - Converts the contained object to a JSON []byte blob formatted with prefix, indent.
+func (g *Container) BytesIndent(prefix string, indent string) []byte {
+	if g.object != nil {
+		if bytes, err := json.MarshalIndent(g.object, prefix, indent); err == nil {
+			return bytes
+		}
+	}
+	return []byte("{}")
+}
+
+// String - Converts the contained object to a JSON formatted string.
+func (g *Container) String() string {
+	return string(g.Bytes())
+}
+
+// StringIndent - Converts the contained object back to a JSON formatted string with prefix, indent.
+func (g *Container) StringIndent(prefix string, indent string) string {
+	return string(g.BytesIndent(prefix, indent))
+}
+
+// EncodeOpt is a functional option for the EncodeJSON method.
+type EncodeOpt func(e *json.Encoder)
+
+// EncodeOptHTMLEscape sets the encoder to escape the JSON for html.
+func EncodeOptHTMLEscape(doEscape bool) EncodeOpt {
+	return func(e *json.Encoder) {
+		e.SetEscapeHTML(doEscape)
+	}
+}
+
+// EncodeOptIndent sets the encoder to indent the JSON output.
+func EncodeOptIndent(prefix string, indent string) EncodeOpt {
+	return func(e *json.Encoder) {
+		e.SetIndent(prefix, indent)
+	}
+}
+
+// EncodeJSON - Encodes the contained object back to a JSON formatted []byte
+// using a variant list of modifier functions for the encoder being used.
+// Functions for modifying the output are prefixed with EncodeOpt, e.g.
+// EncodeOptHTMLEscape.
+func (g *Container) EncodeJSON(encodeOpts ...EncodeOpt) []byte {
+	var b bytes.Buffer
+	encoder := json.NewEncoder(&b)
+	encoder.SetEscapeHTML(false) // Do not escape by default.
+	for _, opt := range encodeOpts {
+		opt(encoder)
+	}
+	if err := encoder.Encode(g.object); err != nil {
+		return []byte("{}")
+	}
+	result := b.Bytes()
+	if len(result) > 0 {
+		result = result[:len(result)-1]
+	}
+	return result
+}
+
+// New - Create a new gabs JSON object.
+func New() *Container {
+	return &Container{map[string]interface{}{}}
+}
+
+// Consume - Gobble up an already converted JSON object, or a fresh map[string]interface{} object.
+func Consume(root interface{}) (*Container, error) {
+	return &Container{root}, nil
+}
+
+// ParseJSON - Convert a string into a representation of the parsed JSON.
+func ParseJSON(sample []byte) (*Container, error) {
+	var gabs Container
+
+	if err := json.Unmarshal(sample, &gabs.object); err != nil {
+		return nil, err
+	}
+
+	return &gabs, nil
+}
+
+// ParseJSONDecoder - Convert a json.Decoder into a representation of the parsed JSON.
+func ParseJSONDecoder(decoder *json.Decoder) (*Container, error) {
+	var gabs Container
+
+	if err := decoder.Decode(&gabs.object); err != nil {
+		return nil, err
+	}
+
+	return &gabs, nil
+}
+
+// ParseJSONFile - Read a file and convert into a representation of the parsed JSON.
+func ParseJSONFile(path string) (*Container, error) {
+	if len(path) > 0 {
+		cBytes, err := ioutil.ReadFile(path)
+		if err != nil {
+			return nil, err
+		}
+
+		container, err := ParseJSON(cBytes)
+		if err != nil {
+			return nil, err
+		}
+
+		return container, nil
+	}
+	return nil, ErrInvalidPath
+}
+
+// ParseJSONBuffer - Read the contents of a buffer into a representation of the parsed JSON.
+func ParseJSONBuffer(buffer io.Reader) (*Container, error) {
+	var gabs Container
+	jsonDecoder := json.NewDecoder(buffer)
+	if err := jsonDecoder.Decode(&gabs.object); err != nil {
+		return nil, err
+	}
+
+	return &gabs, nil
+}
+
+//--------------------------------------------------------------------------------------------------

vendor/github.com/d5/tengo/.gitignore

@@ -0,0 +1 @@
+dist/

vendor/github.com/d5/tengo/.goreleaser.yml

@@ -0,0 +1,23 @@
+builds:
+  - env:
+      - CGO_ENABLED=0
+    main: ./cmd/tengo/main.go
+    goos:
+      - darwin
+      - linux
+      - windows
+  - env:
+      - CGO_ENABLED=0
+    main: ./cmd/tengomin/main.go
+    binary: tengomin
+    goos:
+      - darwin
+      - linux
+      - windows
+archive:
+  files:
+    - none*
+checksum:
+  name_template: 'checksums.txt'
+changelog:
+  sort: asc

vendor/github.com/d5/tengo/.travis.yml

@@ -0,0 +1,17 @@
+language: go
+
+go:
+  - 1.9
+
+install:
+  - go get -u golang.org/x/lint/golint
+
+script:
+  - make test
+
+deploy:
+  - provider: script
+    skip_cleanup: true
+    script: curl -sL https://git.io/goreleaser | bash
+    on:
+      tags: true

vendor/github.com/d5/tengo/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2015 Dmitri Shuralyov
+Copyright (c) 2019 Daniel Kang
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

vendor/github.com/d5/tengo/Makefile

@@ -0,0 +1,11 @@
+vet:
+	go vet ./...
+
+lint:
+	golint -set_exit_status ./...
+
+test: vet lint
+	go test -race -cover ./...
+
+fmt:
+	go fmt ./...

vendor/github.com/d5/tengo/README.md

@@ -0,0 +1,74 @@
+<p align="center">
+  <img src="https://raw.githubusercontent.com/d5/tengolang.com/master/logo_400.png" width="200" height="200">
+</p>
+
+# The Tengo Language
+
+[![GoDoc](https://godoc.org/github.com/d5/tengo?status.svg)](https://godoc.org/github.com/d5/tengo/script)
+[![Go Report Card](https://goreportcard.com/badge/github.com/d5/tengo)](https://goreportcard.com/report/github.com/d5/tengo)
+[![Build Status](https://travis-ci.org/d5/tengo.svg?branch=master)](https://travis-ci.org/d5/tengo)
+[![](https://img.shields.io/badge/Support%20Tengo-%241-brightgreen.svg)](https://www.patreon.com/tengolang)
+
+**Tengo is a small, dynamic, fast, secure script language for Go.** 
+
+Tengo is **[fast](#benchmark)** and secure because it's compiled/executed as bytecode on stack-based VM that's written in native Go.
+
+```golang
+/* The Tengo Language */
+
+each := func(seq, fn) {
+    for x in seq { fn(x) }
+}
+
+sum := func(init, seq) {
+    each(seq, func(x) { init += x })
+    return init
+}
+
+n := sum(0, [1, 2, 3])   // == 6
+s := sum("", [1, 2, 3])  // == "123"
+```
+
+> Run this code in the [Playground](https://tengolang.com/?s=d01cf9ed81daba939e26618530eb171f7397d9c9)
+
+## Features
+
+- Simple and highly readable [Syntax](https://github.com/d5/tengo/blob/master/docs/tutorial.md)
+  - Dynamic typing with type coercion
+  - Higher-order functions and closures
+  - Immutable values
+  - Garbage collection
+- [Securely Embeddable](https://github.com/d5/tengo/blob/master/docs/interoperability.md) and [Extensible](https://github.com/d5/tengo/blob/master/docs/objects.md)
+- Compiler/runtime written in native Go _(no external deps or cgo)_
+- Executable as a [standalone](https://github.com/d5/tengo/blob/master/docs/tengo-cli.md) language / REPL
+
+## Benchmark
+
+| | fib(35) | fibt(35) |  Type  |
+| :--- |    ---: |     ---: |  :---: |
+| Go | `58ms` | `4ms` | Go (native) |
+| [**Tengo**](https://github.com/d5/tengo) | `4,180ms` | `5ms` | VM on Go |
+| Lua | `1,695ms` | `3ms` | Lua (native) |
+| [go-lua](https://github.com/Shopify/go-lua) | `5,163ms` | `5ms` | Lua VM on Go |
+| [GopherLua](https://github.com/yuin/gopher-lua) | `5,525ms` | `5ms` | Lua VM on Go |
+| Python | `3,097ms` | `27ms` | Python (native) |
+| [starlark-go](https://github.com/google/starlark-go) | `15,307ms` | `5ms` | Python-like Interpreter on Go |
+| [gpython](https://github.com/go-python/gpython) | `17,656ms` | `5ms` | Python Interpreter on Go |
+| [goja](https://github.com/dop251/goja) | `6,876ms` | `5ms` | JS VM on Go |
+| [otto](https://github.com/robertkrimen/otto) | `81,886ms` | `12ms` | JS Interpreter on Go |
+| [Anko](https://github.com/mattn/anko) | `97,517ms` | `14ms` | Interpreter on Go |
+
+_* [fib(35)](https://github.com/d5/tengobench/blob/master/code/fib.tengo): Fibonacci(35)_  
+_* [fibt(35)](https://github.com/d5/tengobench/blob/master/code/fibtc.tengo): [tail-call](https://en.wikipedia.org/wiki/Tail_call) version of Fibonacci(35)_  
+_* **Go** does not read the source code from file, while all other cases do_  
+_* See [here](https://github.com/d5/tengobench) for commands/codes used_
+
+## References
+
+- [Language Syntax](https://github.com/d5/tengo/blob/master/docs/tutorial.md)
+- [Object Types](https://github.com/d5/tengo/blob/master/docs/objects.md)
+- [Runtime Types](https://github.com/d5/tengo/blob/master/docs/runtime-types.md) and [Operators](https://github.com/d5/tengo/blob/master/docs/operators.md)
+- [Builtin Functions](https://github.com/d5/tengo/blob/master/docs/builtins.md)
+- [Interoperability](https://github.com/d5/tengo/blob/master/docs/interoperability.md)
+- [Tengo CLI](https://github.com/d5/tengo/blob/master/docs/tengo-cli.md)
+- [Standard Library](https://github.com/d5/tengo/blob/master/docs/stdlib.md)

vendor/github.com/d5/tengo/compiler/ast/array_lit.go

@@ -0,0 +1,35 @@
+package ast
+
+import (
+	"strings"
+
+	"github.com/d5/tengo/compiler/source"
+)
+
+// ArrayLit represents an array literal.
+type ArrayLit struct {
+	Elements []Expr
+	LBrack   source.Pos
+	RBrack   source.Pos
+}
+
+func (e *ArrayLit) exprNode() {}
+
+// Pos returns the position of first character belonging to the node.
+func (e *ArrayLit) Pos() source.Pos {
+	return e.LBrack
+}
+
+// End returns the position of first character immediately after the node.
+func (e *ArrayLit) End() source.Pos {
+	return e.RBrack + 1
+}
+
+func (e *ArrayLit) String() string {
+	var elements []string
+	for _, m := range e.Elements {
+		elements = append(elements, m.String())
+	}
+
+	return "[" + strings.Join(elements, ", ") + "]"
+}

vendor/github.com/d5/tengo/compiler/ast/assign_stmt.go

@@ -0,0 +1,40 @@
+package ast
+
+import (
+	"strings"
+
+	"github.com/d5/tengo/compiler/source"
+	"github.com/d5/tengo/compiler/token"
+)
+
+// AssignStmt represents an assignment statement.
+type AssignStmt struct {
+	LHS      []Expr
+	RHS      []Expr
+	Token    token.Token
+	TokenPos source.Pos
+}
+
+func (s *AssignStmt) stmtNode() {}
+
+// Pos returns the position of first character belonging to the node.
+func (s *AssignStmt) Pos() source.Pos {
+	return s.LHS[0].Pos()
+}
+
+// End returns the position of first character immediately after the node.
+func (s *AssignStmt) End() source.Pos {
+	return s.RHS[len(s.RHS)-1].End()
+}
+
+func (s *AssignStmt) String() string {
+	var lhs, rhs []string
+	for _, e := range s.LHS {
+		lhs = append(lhs, e.String())
+	}
+	for _, e := range s.RHS {
+		rhs = append(rhs, e.String())
+	}
+
+	return strings.Join(lhs, ", ") + " " + s.Token.String() + " " + strings.Join(rhs, ", ")
+}

vendor/github.com/d5/tengo/compiler/ast/ast.go

@@ -0,0 +1,5 @@
+package ast
+
+const (
+	nullRep = "<null>"
+)

vendor/github.com/d5/tengo/compiler/ast/bad_expr.go

@@ -0,0 +1,25 @@
+package ast
+
+import "github.com/d5/tengo/compiler/source"
+
+// BadExpr represents a bad expression.
+type BadExpr struct {
+	From source.Pos
+	To   source.Pos
+}
+
+func (e *BadExpr) exprNode() {}
+
+// Pos returns the position of first character belonging to the node.
+func (e *BadExpr) Pos() source.Pos {
+	return e.From
+}
+
+// End returns the position of first character immediately after the node.
+func (e *BadExpr) End() source.Pos {
+	return e.To
+}
+
+func (e *BadExpr) String() string {
+	return "<bad expression>"
+}

vendor/github.com/d5/tengo/compiler/ast/bad_stmt.go

@@ -0,0 +1,25 @@
+package ast
+
+import "github.com/d5/tengo/compiler/source"
+
+// BadStmt represents a bad statement.
+type BadStmt struct {
+	From source.Pos
+	To   source.Pos
+}
+
+func (s *BadStmt) stmtNode() {}
+
+// Pos returns the position of first character belonging to the node.
+func (s *BadStmt) Pos() source.Pos {
+	return s.From
+}
+
+// End returns the position of first character immediately after the node.
+func (s *BadStmt) End() source.Pos {
+	return s.To
+}
+
+func (s *BadStmt) String() string {
+	return "<bad statement>"
+}

vendor/github.com/d5/tengo/compiler/ast/binary_expr.go

@@ -0,0 +1,30 @@
+package ast
+
+import (
+	"github.com/d5/tengo/compiler/source"
+	"github.com/d5/tengo/compiler/token"
+)
+
+// BinaryExpr represents a binary operator expression.
+type BinaryExpr struct {
+	LHS      Expr
+	RHS      Expr
+	Token    token.Token
+	TokenPos source.Pos
+}
+
+func (e *BinaryExpr) exprNode() {}
+
+// Pos returns the position of first character belonging to the node.
+func (e *BinaryExpr) Pos() source.Pos {
+	return e.LHS.Pos()
+}
+
+// End returns the position of first character immediately after the node.
+func (e *BinaryExpr) End() source.Pos {
+	return e.RHS.End()
+}
+
+func (e *BinaryExpr) String() string {
+	return "(" + e.LHS.String() + " " + e.Token.String() + " " + e.RHS.String() + ")"
+}

vendor/github.com/d5/tengo/compiler/ast/block_stmt.go

@@ -0,0 +1,35 @@
+package ast
+
+import (
+	"strings"
+
+	"github.com/d5/tengo/compiler/source"
+)
+
+// BlockStmt represents a block statement.
+type BlockStmt struct {
+	Stmts  []Stmt
+	LBrace source.Pos
+	RBrace source.Pos
+}
+
+func (s *BlockStmt) stmtNode() {}
+
+// Pos returns the position of first character belonging to the node.
+func (s *BlockStmt) Pos() source.Pos {
+	return s.LBrace
+}
+
+// End returns the position of first character immediately after the node.
+func (s *BlockStmt) End() source.Pos {
+	return s.RBrace + 1
+}
+
+func (s *BlockStmt) String() string {
+	var list []string
+	for _, e := range s.Stmts {
+		list = append(list, e.String())
+	}
+
+	return "{" + strings.Join(list, "; ") + "}"
+}

vendor/github.com/d5/tengo/compiler/ast/bool_lit.go

@@ -0,0 +1,26 @@
+package ast
+
+import "github.com/d5/tengo/compiler/source"
+
+// BoolLit represents a boolean literal.
+type BoolLit struct {
+	Value    bool
+	ValuePos source.Pos
+	Literal  string
+}
+
+func (e *BoolLit) exprNode() {}
+
+// Pos returns the position of first character belonging to the node.
+func (e *BoolLit) Pos() source.Pos {
+	return e.ValuePos
+}
+
+// End returns the position of first character immediately after the node.
+func (e *BoolLit) End() source.Pos {
+	return source.Pos(int(e.ValuePos) + len(e.Literal))
+}
+
+func (e *BoolLit) String() string {
+	return e.Literal
+}

vendor/github.com/d5/tengo/compiler/ast/branch_stmt.go

@@ -0,0 +1,38 @@
+package ast
+
+import (
+	"github.com/d5/tengo/compiler/source"
+	"github.com/d5/tengo/compiler/token"
+)
+
+// BranchStmt represents a branch statement.
+type BranchStmt struct {
+	Token    token.Token
+	TokenPos source.Pos
+	Label    *Ident
+}
+
+func (s *BranchStmt) stmtNode() {}
+
+// Pos returns the position of first character belonging to the node.
+func (s *BranchStmt) Pos() source.Pos {
+	return s.TokenPos
+}
+
+// End returns the position of first character immediately after the node.
+func (s *BranchStmt) End() source.Pos {
+	if s.Label != nil {
+		return s.Label.End()
+	}
+
+	return source.Pos(int(s.TokenPos) + len(s.Token.String()))
+}
+
+func (s *BranchStmt) String() string {
+	var label string
+	if s.Label != nil {
+		label = " " + s.Label.Name
+	}
+
+	return s.Token.String() + label
+}

vendor/github.com/d5/tengo/compiler/ast/call_expr.go

@@ -0,0 +1,36 @@
+package ast
+
+import (
+	"strings"
+
+	"github.com/d5/tengo/compiler/source"
+)
+
+// CallExpr represents a function call expression.
+type CallExpr struct {
+	Func   Expr
+	LParen source.Pos
+	Args   []Expr
+	RParen source.Pos
+}
+
+func (e *CallExpr) exprNode() {}
+
+// Pos returns the position of first character belonging to the node.
+func (e *CallExpr) Pos() source.Pos {
+	return e.Func.Pos()
+}
+
+// End returns the position of first character immediately after the node.
+func (e *CallExpr) End() source.Pos {
+	return e.RParen + 1
+}
+
+func (e *CallExpr) String() string {
+	var args []string
+	for _, e := range e.Args {
+		args = append(args, e.String())
+	}
+
+	return e.Func.String() + "(" + strings.Join(args, ", ") + ")"
+}

vendor/github.com/d5/tengo/compiler/ast/char_lit.go

@@ -0,0 +1,26 @@
+package ast
+
+import "github.com/d5/tengo/compiler/source"
+
+// CharLit represents a character literal.
+type CharLit struct {
+	Value    rune
+	ValuePos source.Pos
+	Literal  string
+}
+
+func (e *CharLit) exprNode() {}
+
+// Pos returns the position of first character belonging to the node.
+func (e *CharLit) Pos() source.Pos {
+	return e.ValuePos
+}
+
+// End returns the position of first character immediately after the node.
+func (e *CharLit) End() source.Pos {
+	return source.Pos(int(e.ValuePos) + len(e.Literal))
+}
+
+func (e *CharLit) String() string {
+	return e.Literal
+}

vendor/github.com/d5/tengo/compiler/ast/cond_expr.go

@@ -0,0 +1,30 @@
+package ast
+
+import (
+	"github.com/d5/tengo/compiler/source"
+)
+
+// CondExpr represents a ternary conditional expression.
+type CondExpr struct {
+	Cond        Expr
+	True        Expr
+	False       Expr
+	QuestionPos source.Pos
+	ColonPos    source.Pos
+}
+
+func (e *CondExpr) exprNode() {}
+
+// Pos returns the position of first character belonging to the node.
+func (e *CondExpr) Pos() source.Pos {
+	return e.Cond.Pos()
+}
+
+// End returns the position of first character immediately after the node.
+func (e *CondExpr) End() source.Pos {
+	return e.False.End()
+}
+
+func (e *CondExpr) String() string {
+	return "(" + e.Cond.String() + " ? " + e.True.String() + " : " + e.False.String() + ")"
+}

vendor/github.com/d5/tengo/compiler/ast/empty_stmt.go

@@ -0,0 +1,29 @@
+package ast
+
+import "github.com/d5/tengo/compiler/source"
+
+// EmptyStmt represents an empty statement.
+type EmptyStmt struct {
+	Semicolon source.Pos
+	Implicit  bool
+}
+
+func (s *EmptyStmt) stmtNode() {}
+
+// Pos returns the position of first character belonging to the node.
+func (s *EmptyStmt) Pos() source.Pos {
+	return s.Semicolon
+}
+
+// End returns the position of first character immediately after the node.
+func (s *EmptyStmt) End() source.Pos {
+	if s.Implicit {
+		return s.Semicolon
+	}
+
+	return s.Semicolon + 1
+}
+
+func (s *EmptyStmt) String() string {
+	return ";"
+}

vendor/github.com/d5/tengo/compiler/ast/error_expr.go

@@ -0,0 +1,29 @@
+package ast
+
+import (
+	"github.com/d5/tengo/compiler/source"
+)
+
+// ErrorExpr represents an error expression
+type ErrorExpr struct {
+	Expr     Expr
+	ErrorPos source.Pos
+	LParen   source.Pos
+	RParen   source.Pos
+}
+
+func (e *ErrorExpr) exprNode() {}
+
+// Pos returns the position of first character belonging to the node.
+func (e *ErrorExpr) Pos() source.Pos {
+	return e.ErrorPos
+}
+
+// End returns the position of first character immediately after the node.
+func (e *ErrorExpr) End() source.Pos {
+	return e.RParen
+}
+
+func (e *ErrorExpr) String() string {
+	return "error(" + e.Expr.String() + ")"
+}

vendor/github.com/d5/tengo/compiler/ast/export_stmt.go

@@ -0,0 +1,27 @@
+package ast
+
+import (
+	"github.com/d5/tengo/compiler/source"
+)
+
+// ExportStmt represents an export statement.
+type ExportStmt struct {
+	ExportPos source.Pos
+	Result    Expr
+}
+
+func (s *ExportStmt) stmtNode() {}
+
+// Pos returns the position of first character belonging to the node.
+func (s *ExportStmt) Pos() source.Pos {
+	return s.ExportPos
+}
+
+// End returns the position of first character immediately after the node.
+func (s *ExportStmt) End() source.Pos {
+	return s.Result.End()
+}
+
+func (s *ExportStmt) String() string {
+	return "export " + s.Result.String()
+}

vendor/github.com/d5/tengo/compiler/ast/expr.go

@@ -0,0 +1,7 @@
+package ast
+
+// Expr represents an expression node in the AST.
+type Expr interface {
+	Node
+	exprNode()
+}

vendor/github.com/d5/tengo/compiler/ast/expr_stmt.go

@@ -0,0 +1,24 @@
+package ast
+
+import "github.com/d5/tengo/compiler/source"
+
+// ExprStmt represents an expression statement.
+type ExprStmt struct {
+	Expr Expr
+}
+
+func (s *ExprStmt) stmtNode() {}
+
+// Pos returns the position of first character belonging to the node.
+func (s *ExprStmt) Pos() source.Pos {
+	return s.Expr.Pos()
+}
+
+// End returns the position of first character immediately after the node.
+func (s *ExprStmt) End() source.Pos {
+	return s.Expr.End()
+}
+
+func (s *ExprStmt) String() string {
+	return s.Expr.String()
+}

vendor/github.com/d5/tengo/compiler/ast/file.go

@@ -0,0 +1,32 @@
+package ast
+
+import (
+	"strings"
+
+	"github.com/d5/tengo/compiler/source"
+)
+
+// File represents a file unit.
+type File struct {
+	InputFile *source.File
+	Stmts     []Stmt
+}
+
+// Pos returns the position of first character belonging to the node.
+func (n *File) Pos() source.Pos {
+	return source.Pos(n.InputFile.Base)
+}
+
+// End returns the position of first character immediately after the node.
+func (n *File) End() source.Pos {
+	return source.Pos(n.InputFile.Base + n.InputFile.Size)
+}
+
+func (n *File) String() string {
+	var stmts []string
+	for _, e := range n.Stmts {
+		stmts = append(stmts, e.String())
+	}
+
+	return strings.Join(stmts, "; ")
+}

vendor/github.com/d5/tengo/compiler/ast/float_lit.go

@@ -0,0 +1,26 @@
+package ast
+
+import "github.com/d5/tengo/compiler/source"
+
+// FloatLit represents a floating point literal.
+type FloatLit struct {
+	Value    float64
+	ValuePos source.Pos
+	Literal  string
+}
+
+func (e *FloatLit) exprNode() {}
+
+// Pos returns the position of first character belonging to the node.
+func (e *FloatLit) Pos() source.Pos {
+	return e.ValuePos
+}
+
+// End returns the position of first character immediately after the node.
+func (e *FloatLit) End() source.Pos {
+	return source.Pos(int(e.ValuePos) + len(e.Literal))
+}
+
+func (e *FloatLit) String() string {
+	return e.Literal
+}

vendor/github.com/d5/tengo/compiler/ast/for_in_stmt.go

@@ -0,0 +1,32 @@
+package ast
+
+import "github.com/d5/tengo/compiler/source"
+
+// ForInStmt represents a for-in statement.
+type ForInStmt struct {
+	ForPos   source.Pos
+	Key      *Ident
+	Value    *Ident
+	Iterable Expr
+	Body     *BlockStmt
+}
+
+func (s *ForInStmt) stmtNode() {}
+
+// Pos returns the position of first character belonging to the node.
+func (s *ForInStmt) Pos() source.Pos {
+	return s.ForPos
+}
+
+// End returns the position of first character immediately after the node.
+func (s *ForInStmt) End() source.Pos {
+	return s.Body.End()
+}
+
+func (s *ForInStmt) String() string {
+	if s.Value != nil {
+		return "for " + s.Key.String() + ", " + s.Value.String() + " in " + s.Iterable.String() + " " + s.Body.String()
+	}
+
+	return "for " + s.Key.String() + " in " + s.Iterable.String() + " " + s.Body.String()
+}

vendor/github.com/d5/tengo/compiler/ast/for_stmt.go

@@ -0,0 +1,43 @@
+package ast
+
+import "github.com/d5/tengo/compiler/source"
+
+// ForStmt represents a for statement.
+type ForStmt struct {
+	ForPos source.Pos
+	Init   Stmt
+	Cond   Expr
+	Post   Stmt
+	Body   *BlockStmt
+}
+
+func (s *ForStmt) stmtNode() {}
+
+// Pos returns the position of first character belonging to the node.
+func (s *ForStmt) Pos() source.Pos {
+	return s.ForPos
+}
+
+// End returns the position of first character immediately after the node.
+func (s *ForStmt) End() source.Pos {
+	return s.Body.End()
+}
+
+func (s *ForStmt) String() string {
+	var init, cond, post string
+	if s.Init != nil {
+		init = s.Init.String()
+	}
+	if s.Cond != nil {
+		cond = s.Cond.String() + " "
+	}
+	if s.Post != nil {
+		post = s.Post.String()
+	}
+
+	if init != "" || post != "" {
+		return "for " + init + " ; " + cond + " ; " + post + s.Body.String()
+	}
+
+	return "for " + cond + s.Body.String()
+}

vendor/github.com/d5/tengo/compiler/ast/func_lit.go

@@ -0,0 +1,25 @@
+package ast
+
+import "github.com/d5/tengo/compiler/source"
+
+// FuncLit represents a function literal.
+type FuncLit struct {
+	Type *FuncType
+	Body *BlockStmt
+}
+
+func (e *FuncLit) exprNode() {}
+
+// Pos returns the position of first character belonging to the node.
+func (e *FuncLit) Pos() source.Pos {
+	return e.Type.Pos()
+}
+
+// End returns the position of first character immediately after the node.
+func (e *FuncLit) End() source.Pos {
+	return e.Body.End()
+}
+
+func (e *FuncLit) String() string {
+	return "func" + e.Type.Params.String() + " " + e.Body.String()
+}

vendor/github.com/d5/tengo/compiler/ast/func_type.go

@@ -0,0 +1,25 @@
+package ast
+
+import "github.com/d5/tengo/compiler/source"
+
+// FuncType represents a function type definition.
+type FuncType struct {
+	FuncPos source.Pos
+	Params  *IdentList
+}
+
+func (e *FuncType) exprNode() {}
+
+// Pos returns the position of first character belonging to the node.
+func (e *FuncType) Pos() source.Pos {
+	return e.FuncPos
+}
+
+// End returns the position of first character immediately after the node.
+func (e *FuncType) End() source.Pos {
+	return e.Params.End()
+}
+
+func (e *FuncType) String() string {
+	return "func" + e.Params.String()
+}

vendor/github.com/d5/tengo/compiler/ast/ident.go

@@ -0,0 +1,29 @@
+package ast
+
+import "github.com/d5/tengo/compiler/source"
+
+// Ident represents an identifier.
+type Ident struct {
+	Name    string
+	NamePos source.Pos
+}
+
+func (e *Ident) exprNode() {}
+
+// Pos returns the position of first character belonging to the node.
+func (e *Ident) Pos() source.Pos {
+	return e.NamePos
+}
+
+// End returns the position of first character immediately after the node.
+func (e *Ident) End() source.Pos {
+	return source.Pos(int(e.NamePos) + len(e.Name))
+}
+
+func (e *Ident) String() string {
+	if e != nil {
+		return e.Name
+	}
+
+	return nullRep
+}

vendor/github.com/d5/tengo/compiler/ast/ident_list.go

@@ -0,0 +1,58 @@
+package ast
+
+import (
+	"strings"
+
+	"github.com/d5/tengo/compiler/source"
+)
+
+// IdentList represents a list of identifiers.
+type IdentList struct {
+	LParen source.Pos
+	List   []*Ident
+	RParen source.Pos
+}
+
+// Pos returns the position of first character belonging to the node.
+func (n *IdentList) Pos() source.Pos {
+	if n.LParen.IsValid() {
+		return n.LParen
+	}
+
+	if len(n.List) > 0 {
+		return n.List[0].Pos()
+	}
+
+	return source.NoPos
+}
+
+// End returns the position of first character immediately after the node.
+func (n *IdentList) End() source.Pos {
+	if n.RParen.IsValid() {
+		return n.RParen + 1
+	}
+
+	if l := len(n.List); l > 0 {
+		return n.List[l-1].End()
+	}
+
+	return source.NoPos
+}
+
+// NumFields returns the number of fields.
+func (n *IdentList) NumFields() int {
+	if n == nil {
+		return 0
+	}
+
+	return len(n.List)
+}
+
+func (n *IdentList) String() string {
+	var list []string
+	for _, e := range n.List {
+		list = append(list, e.String())
+	}
+
+	return "(" + strings.Join(list, ", ") + ")"
+}

vendor/github.com/d5/tengo/compiler/ast/if_stmt.go

@@ -0,0 +1,40 @@
+package ast
+
+import "github.com/d5/tengo/compiler/source"
+
+// IfStmt represents an if statement.
+type IfStmt struct {
+	IfPos source.Pos
+	Init  Stmt
+	Cond  Expr
+	Body  *BlockStmt
+	Else  Stmt // else branch; or nil
+}
+
+func (s *IfStmt) stmtNode() {}
+
+// Pos returns the position of first character belonging to the node.
+func (s *IfStmt) Pos() source.Pos {
+	return s.IfPos
+}
+
+// End returns the position of first character immediately after the node.
+func (s *IfStmt) End() source.Pos {
+	if s.Else != nil {
+		return s.Else.End()
+	}
+
+	return s.Body.End()
+}
+
+func (s *IfStmt) String() string {
+	var initStmt, elseStmt string
+	if s.Init != nil {
+		initStmt = s.Init.String() + "; "
+	}
+	if s.Else != nil {
+		elseStmt = " else " + s.Else.String()
+	}
+
+	return "if " + initStmt + s.Cond.String() + " " + s.Body.String() + elseStmt
+}

vendor/github.com/d5/tengo/compiler/ast/immutable_expr.go

@@ -0,0 +1,29 @@
+package ast
+
+import (
+	"github.com/d5/tengo/compiler/source"
+)
+
+// ImmutableExpr represents an immutable expression
+type ImmutableExpr struct {
+	Expr     Expr
+	ErrorPos source.Pos
+	LParen   source.Pos
+	RParen   source.Pos
+}
+
+func (e *ImmutableExpr) exprNode() {}
+
+// Pos returns the position of first character belonging to the node.
+func (e *ImmutableExpr) Pos() source.Pos {
+	return e.ErrorPos
+}
+
+// End returns the position of first character immediately after the node.
+func (e *ImmutableExpr) End() source.Pos {
+	return e.RParen
+}
+
+func (e *ImmutableExpr) String() string {
+	return "immutable(" + e.Expr.String() + ")"
+}

vendor/github.com/d5/tengo/compiler/ast/import_expr.go

@@ -0,0 +1,29 @@
+package ast
+
+import (
+	"github.com/d5/tengo/compiler/source"
+	"github.com/d5/tengo/compiler/token"
+)
+
+// ImportExpr represents an import expression
+type ImportExpr struct {
+	ModuleName string
+	Token      token.Token
+	TokenPos   source.Pos
+}
+
+func (e *ImportExpr) exprNode() {}
+
+// Pos returns the position of first character belonging to the node.
+func (e *ImportExpr) Pos() source.Pos {
+	return e.TokenPos
+}
+
+// End returns the position of first character immediately after the node.
+func (e *ImportExpr) End() source.Pos {
+	return source.Pos(int(e.TokenPos) + 10 + len(e.ModuleName)) // import("moduleName")
+}
+
+func (e *ImportExpr) String() string {
+	return `import("` + e.ModuleName + `")"`
+}

vendor/github.com/d5/tengo/compiler/ast/inc_dec_stmt.go

@@ -0,0 +1,29 @@
+package ast
+
+import (
+	"github.com/d5/tengo/compiler/source"
+	"github.com/d5/tengo/compiler/token"
+)
+
+// IncDecStmt represents increment or decrement statement.
+type IncDecStmt struct {
+	Expr     Expr
+	Token    token.Token
+	TokenPos source.Pos
+}
+
+func (s *IncDecStmt) stmtNode() {}
+
+// Pos returns the position of first character belonging to the node.
+func (s *IncDecStmt) Pos() source.Pos {
+	return s.Expr.Pos()
+}
+
+// End returns the position of first character immediately after the node.
+func (s *IncDecStmt) End() source.Pos {
+	return source.Pos(int(s.TokenPos) + 2)
+}
+
+func (s *IncDecStmt) String() string {
+	return s.Expr.String() + s.Token.String()
+}

vendor/github.com/d5/tengo/compiler/ast/index_expr.go

@@ -0,0 +1,32 @@
+package ast
+
+import "github.com/d5/tengo/compiler/source"
+
+// IndexExpr represents an index expression.
+type IndexExpr struct {
+	Expr   Expr
+	LBrack source.Pos
+	Index  Expr
+	RBrack source.Pos
+}
+
+func (e *IndexExpr) exprNode() {}
+
+// Pos returns the position of first character belonging to the node.
+func (e *IndexExpr) Pos() source.Pos {
+	return e.Expr.Pos()
+}
+
+// End returns the position of first character immediately after the node.
+func (e *IndexExpr) End() source.Pos {
+	return e.RBrack + 1
+}
+
+func (e *IndexExpr) String() string {
+	var index string
+	if e.Index != nil {
+		index = e.Index.String()
+	}
+
+	return e.Expr.String() + "[" + index + "]"
+}

vendor/github.com/d5/tengo/compiler/ast/int_lit.go

@@ -0,0 +1,26 @@
+package ast
+
+import "github.com/d5/tengo/compiler/source"
+
+// IntLit represents an integer literal.
+type IntLit struct {
+	Value    int64
+	ValuePos source.Pos
+	Literal  string
+}
+
+func (e *IntLit) exprNode() {}
+
+// Pos returns the position of first character belonging to the node.
+func (e *IntLit) Pos() source.Pos {
+	return e.ValuePos
+}
+
+// End returns the position of first character immediately after the node.
+func (e *IntLit) End() source.Pos {
+	return source.Pos(int(e.ValuePos) + len(e.Literal))
+}
+
+func (e *IntLit) String() string {
+	return e.Literal
+}

vendor/github.com/d5/tengo/compiler/ast/map_element_lit.go

@@ -0,0 +1,27 @@
+package ast
+
+import "github.com/d5/tengo/compiler/source"
+
+// MapElementLit represents a map element.
+type MapElementLit struct {
+	Key      string
+	KeyPos   source.Pos
+	ColonPos source.Pos
+	Value    Expr
+}
+
+func (e *MapElementLit) exprNode() {}
+
+// Pos returns the position of first character belonging to the node.
+func (e *MapElementLit) Pos() source.Pos {
+	return e.KeyPos
+}
+
+// End returns the position of first character immediately after the node.
+func (e *MapElementLit) End() source.Pos {
+	return e.Value.End()
+}
+
+func (e *MapElementLit) String() string {
+	return e.Key + ": " + e.Value.String()
+}

vendor/github.com/d5/tengo/compiler/ast/map_lit.go

@@ -0,0 +1,35 @@
+package ast
+
+import (
+	"strings"
+
+	"github.com/d5/tengo/compiler/source"
+)
+
+// MapLit represents a map literal.
+type MapLit struct {
+	LBrace   source.Pos
+	Elements []*MapElementLit
+	RBrace   source.Pos
+}
+
+func (e *MapLit) exprNode() {}
+
+// Pos returns the position of first character belonging to the node.
+func (e *MapLit) Pos() source.Pos {
+	return e.LBrace
+}
+
+// End returns the position of first character immediately after the node.
+func (e *MapLit) End() source.Pos {
+	return e.RBrace + 1
+}
+
+func (e *MapLit) String() string {
+	var elements []string
+	for _, m := range e.Elements {
+		elements = append(elements, m.String())
+	}
+
+	return "{" + strings.Join(elements, ", ") + "}"
+}

vendor/github.com/d5/tengo/compiler/ast/node.go

@@ -0,0 +1,13 @@
+package ast
+
+import "github.com/d5/tengo/compiler/source"
+
+// Node represents a node in the AST.
+type Node interface {
+	// Pos returns the position of first character belonging to the node.
+	Pos() source.Pos
+	// End returns the position of first character immediately after the node.
+	End() source.Pos
+	// String returns a string representation of the node.
+	String() string
+}

vendor/github.com/d5/tengo/compiler/ast/paren_expr.go

@@ -0,0 +1,26 @@
+package ast
+
+import "github.com/d5/tengo/compiler/source"
+
+// ParenExpr represents a parenthesis wrapped expression.
+type ParenExpr struct {
+	Expr   Expr
+	LParen source.Pos
+	RParen source.Pos
+}
+
+func (e *ParenExpr) exprNode() {}
+
+// Pos returns the position of first character belonging to the node.
+func (e *ParenExpr) Pos() source.Pos {
+	return e.LParen
+}
+
+// End returns the position of first character immediately after the node.
+func (e *ParenExpr) End() source.Pos {
+	return e.RParen + 1
+}
+
+func (e *ParenExpr) String() string {
+	return "(" + e.Expr.String() + ")"
+}