Release v1.18.1 (#1207)

This is half a fix for #874

This patch introduces a new config flag:
- MediaConvertTgs

These need to be treated independently from the existing
MediaConvertWebPToPNG flag because Tgs→WebP results in an
*animated* WebP, and the WebP→PNG converter can't handle
animated WebP files yet.

Furthermore, some platforms (like discord) don't even support
animated WebP files, so the user may want to fall back to
static PNGs (not APNGs).

The final reason why this is only half a fix is that this
introduces an external dependency, namely lottie, to be
installed like this:

$ pip3 install lottie cairosvg

This patch works by writing the tgs to a temporary file in /tmp,
calling lottie to convert it (this conversion may take several seconds!),
and then deleting the temporary file.
The temporary file is absolutely necessary, as lottie refuses to
work on non-seekable files.
If anyone comes up with a reasonable use case where /tmp is
unavailable, I can add yet another config option for that, if desired.

Telegram will bail out if the option is configured but lottie isn't found.

.fixmie.yml

@@ -0,0 +1,3 @@
+go:
+  comments:
+    disabled: true

.github/ISSUE_TEMPLATE/Bug_report.md

@@ -1,6 +1,7 @@
 ---
 name: Bug report
 about: Create a report to help us improve. (Check the FAQ on the wiki first)
+labels: bug
 
 ---
 

.github/ISSUE_TEMPLATE/Feature_request.md

@@ -1,6 +1,7 @@
 ---
 name: Feature request
 about: Suggest an idea for this project
+labels: enhancement
 
 ---
 

.github/workflows/development.yml

@@ -0,0 +1,57 @@
+name: Development
+on: [push, pull_request]
+jobs:
+  lint:
+    name: golangci-lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 20
+      - name: Run golangci-lint
+        uses: golangci/golangci-lint-action@v2
+        with:
+          version: v1.29
+          args: "-v --new-from-rev HEAD~5"
+  test-build-upload:
+    strategy:
+      matrix:
+        go-version: [1.14.x, 1.15.x]
+        platform: [ubuntu-latest]
+    runs-on: ${{ matrix.platform }}
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+    - name: Checkout code
+      uses: actions/checkout@v2
+      with:
+          fetch-depth: 0
+    - name: Test
+      run: go test ./... -mod=vendor
+    - name: Build
+      run: |
+        mkdir -p output/{win,lin,arm,mac}
+        VERSION=$(git describe --tags)
+        GOOS=linux GOARCH=amd64 go build -mod=vendor -ldflags "-s -X main.githash=$(git log --pretty=format:'%h' -n 1)" -o output/lin/matterbridge-$VERSION-linux-amd64
+        GOOS=windows GOARCH=amd64 go build -mod=vendor -ldflags "-s -X main.githash=$(git log --pretty=format:'%h' -n 1)" -o output/win/matterbridge-$VERSION-windows-amd64.exe
+        GOOS=darwin GOARCH=amd64 go build -mod=vendor -ldflags "-s -X main.githash=$(git log --pretty=format:'%h' -n 1)" -o output/mac/matterbridge-$VERSION-darwin-amd64
+    - name: Upload linux 64-bit
+      if: startsWith(matrix.go-version,'1.15')
+      uses: actions/upload-artifact@v2
+      with:
+        name: matterbridge-linux-64bit
+        path: output/lin
+    - name: Upload windows 64-bit
+      if: startsWith(matrix.go-version,'1.15')
+      uses: actions/upload-artifact@v2
+      with:
+        name: matterbridge-windows-64bit
+        path: output/win
+    - name: Upload darwin 64-bit
+      if: startsWith(matrix.go-version,'1.15')
+      uses: actions/upload-artifact@v2
+      with:
+        name: matterbridge-darwin-64bit
+        path: output/mac

.gitignore

@@ -0,0 +1,6 @@
+# Exclude matterbridge binary
+/matterbridge
+/matterbridge.exe
+
+# Exclude configuration file
+matterbridge.toml

.golangci.yaml

@@ -23,7 +23,7 @@ run:
   # default value is empty list, but next dirs are always skipped independently
   # from this option's value:
   #   	vendor$, third_party$, testdata$, examples$, Godeps$, builtin$
-  skip-dirs:
+  skip-dirs: gateway/bridgemap$
 
   # which files to skip: they will be analyzed, but issues from them
   # won't be reported. Default value is empty list, but there is
@@ -174,7 +174,15 @@ linters:
     - lll
     - maligned
     - prealloc
-
+    - wsl
+    - gomnd
+    - godox
+    - goerr113
+    - testpackage
+    - godot
+    - interfacer
+    - goheader
+    - noctx
 
 # rules to deal with reported isues
 issues:

.goreleaser.yml

@@ -21,14 +21,18 @@ builds:
   ldflags:
     - -s -w -X main.githash={{.ShortCommit}}
 
-archive:
-  name_template: "{{ .Binary }}-{{ .Version }}-{{ .Os }}-{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}"
-  format: binary
-  files:
-    - none*
-  replacements:
-    386: 32bit
-    amd64: 64bit
+archives:
+  -
+    id: matterbridge
+    builds:
+    - matterbridge
+    name_template: "{{ .Binary }}-{{ .Version }}-{{ .Os }}-{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}"
+    format: binary
+    files:
+      - none*
+    replacements:
+      386: 32bit
+      amd64: 64bit
 
 checksum:
   name_template: 'checksums.txt'

.travis.yml

@@ -1,55 +0,0 @@
-language: go
-go_import_path: github.com/42wim/matterbridge
-
-# We have everything vendored so this helps TravisCI not run `go get ...`.
-install: true
-
-git:
-  depth: 200
-
-notifications:
-  email: false
-
-branches:
-  only:
-  - master
-
-jobs:
-  include:
-  - stage: lint
-    # Run linting in one Go environment only.
-    script: ./ci/lint.sh
-    go: 1.12.x
-    env:
-    - GO111MODULE=on
-    - GOLANGCI_VERSION="v1.16.0"
-  - stage: test
-    # Run tests in a combination of Go environments.
-    script: ./ci/test.sh
-    go: 1.11.x
-    env:
-    - GO111MODULE=off
-  - script: ./ci/test.sh
-    go: 1.11.x
-    env:
-    - GO111MODULE=on
-  - script: ./ci/test.sh
-    go: 1.12.x
-    env:
-    - GO111MODULE=on
-    - REPORT_COVERAGE=1
-    - BINDEPLOY=1
-
-before_deploy: /bin/bash ci/bintray.sh
-
-deploy:
-  on:
-    all_branches: true
-    condition: $BINDEPLOY = 1
-  provider: bintray
-  edge:
-    branch: v1.8.47
-  file: ci/deploy.json
-  user: 42wim
-  key:
-    secure: "CeXXe6JOmt7HYR81MdWLua0ltQHhDdkIeRGBFbgd7hkb1wi8eF9DgpAcQrTso8NIlHNZmSAP46uhFgsRvkuezzX0ygalZ7DCJyAyn3sAMEh+UQSHV1WGThRehTtidqRGjetzsIGSwdrJOWil+XTfbO1Z8DGzfakhSuAZka8CM4BAoe3YeP9rYK8h+84x0GHfczvsLtXZ3mWLvQuwe4pK6+ItBCUg0ae7O7ZUpWHy0xQQkkWztY/6RAzXfaG7DuGjIw+20fhx3WOXRNpHCtZ6Bc3qERCpk0s1HhlQWlrN9wDaFTBWYwlvSnNgvxxMbNXJ6RrRJ0l0bA7FUswYwyroxhzrGLdzWDg8dHaQkypocngdalfhpsnoO9j3ApJhomUFJ3UoEq5nOGRUrKn8MPi+dP0zE4kNQ3e4VNa1ufNrvfpWolMg3xh8OXuhQdD5wIM5zFAbRJLqWSCVAjPq4DDPecmvXBOlIial7oa312lN5qnBnUjvAcxszZ+FUyDHT1Grxzna4tMwxY9obPzZUzm7359AOCCwIQFVB8GLqD2nwIstcXS0zGRz+fhviPipHuBa02q5bGUZwmkvrSNab0s8Jo7pCrel2Rz3nWPKaiCfq2WjbW1CLheSMkOQrjsdUd1hhbqNWFPUjJPInTc77NAKCfm5runv5uyowRLh4NNd0sI="

Dockerfile

@@ -1,11 +1,16 @@
-FROM alpine:edge
-ENTRYPOINT ["/bin/matterbridge"]
+FROM alpine:edge AS builder
 
 COPY . /go/src/github.com/42wim/matterbridge
-RUN apk update && apk add go git gcc musl-dev ca-certificates \
+RUN apk update && apk add go git gcc musl-dev \
         && cd /go/src/github.com/42wim/matterbridge \
         && export GOPATH=/go \
         && go get \
-        && go build -x -ldflags "-X main.githash=$(git log --pretty=format:'%h' -n 1)" -o /bin/matterbridge \
-        && rm -rf /go \
-        && apk del --purge git go gcc musl-dev
+        && go build -x -ldflags "-X main.githash=$(git log --pretty=format:'%h' -n 1)" -o /bin/matterbridge
+
+FROM alpine:edge
+RUN apk --no-cache add ca-certificates mailcap
+COPY --from=builder /bin/matterbridge /bin/matterbridge
+RUN mkdir /etc/matterbridge \
+  && touch /etc/matterbridge/matterbridge.toml \
+  && ln -sf /matterbridge.toml /etc/matterbridge/matterbridge.toml
+ENTRYPOINT ["/bin/matterbridge", "-conf", "/etc/matterbridge/matterbridge.toml"]

README.md

@@ -3,32 +3,35 @@
 # matterbridge
 
 ![Matterbridge Logo](img/matterbridge-notext.gif)<br />
-   **A simple chat bridge**<br />
-   Letting people be where they want to be.<br />
-   <sub>Bridges between a growing number of protocols. Click below to demo or join the development chat.</sub>
+**A simple chat bridge**<br />
+Letting people be where they want to be.<br />
+<sub>Bridges between a growing number of protocols. Click below to demo or join the development chat.</sub>
 
    <sup>
 
-   [Gitter][mb-gitter] |
-   [IRC][mb-irc] |
-      [Discord][mb-discord] |
-      [Matrix][mb-matrix] |
-      [Slack][mb-slack] |
-      [Mattermost][mb-mattermost] |
-      [Rocket.Chat][mb-rocketchat] |
-      [XMPP][mb-xmpp] |
-      [Twitch][mb-twitch] |
-      [WhatsApp][mb-whatsapp] |
-      [Zulip][mb-zulip] |
-      [Telegram][mb-telegram] |
-      And more...
-   </sup>
+[Discord][mb-discord] |
+[Gitter][mb-gitter] |
+[IRC][mb-irc] |
+[Keybase][mb-keybase] |
+[Matrix][mb-matrix] |
+[Mattermost][mb-mattermost] |
+[MSTeams][mb-msteams] |
+[Rocket.Chat][mb-rocketchat] |
+[Slack][mb-slack] |
+[Telegram][mb-telegram] |
+[Twitch][mb-twitch] |
+[WhatsApp][mb-whatsapp] |
+[XMPP][mb-xmpp] |
+[Zulip][mb-zulip] |
+And more...
+</sup>
+
+---
 
-----
 [![Download stable](https://img.shields.io/github/release/42wim/matterbridge.svg?label=download%20stable)](https://github.com/42wim/matterbridge/releases/latest)
-   [![Download dev](https://img.shields.io/bintray/v/42wim/nightly/Matterbridge.svg?label=download%20dev&colorB=007ec6)](https://bintray.com/42wim/nightly/Matterbridge/_latestVersion)
-   [![Maintainability](https://api.codeclimate.com/v1/badges/82dff70ef2ba85a6173a/maintainability)](https://codeclimate.com/github/42wim/matterbridge/maintainability)
-   [![Test Coverage](https://api.codeclimate.com/v1/badges/82dff70ef2ba85a6173a/test_coverage)](https://codeclimate.com/github/42wim/matterbridge/test_coverage)<br />
+[![Maintainability](https://api.codeclimate.com/v1/badges/82dff70ef2ba85a6173a/maintainability)](https://codeclimate.com/github/42wim/matterbridge/maintainability)
+[![Test Coverage](https://api.codeclimate.com/v1/badges/82dff70ef2ba85a6173a/test_coverage)](https://codeclimate.com/github/42wim/matterbridge/test_coverage)<br />
+
   <hr />
 </div>
 <div align="right"><sup>
@@ -41,126 +44,161 @@
   </a>
 </p>
 
-### Table of Contents
- * [Features](https://github.com/42wim/matterbridge/wiki/Features)
-   * [Natively supported](#natively-supported)
-   * [3rd party via matterbridge api](#3rd-party-via-matterbridge-api)
-   * [API](#API)
- * [Chat with us](#chat-with-us)
- * [Screenshots](https://github.com/42wim/matterbridge/wiki/)
- * [Installing](#installing)
-   * [Binaries](#binaries)
-   * [Building](#building)
- * [Configuration](#configuration)
-   * [Howto](https://github.com/42wim/matterbridge/wiki/How-to-create-your-config)
-   * [Examples](#examples)
- * [Running](#running)
-   * [Docker](#docker)
- * [Changelog](#changelog)
- * [FAQ](#faq)
- * [Related projects](#related-projects)
- * [Articles](#articles)
- * [Thanks](#thanks)
+# Table of Contents
+
+- [matterbridge](#matterbridge)
+- [Table of Contents](#table-of-contents)
+  - [Features](#features)
+    - [Natively supported](#natively-supported)
+    - [3rd party via matterbridge api](#3rd-party-via-matterbridge-api)
+    - [API](#api)
+  - [Chat with us](#chat-with-us)
+  - [Screenshots](#screenshots)
+  - [Installing / upgrading](#installing--upgrading)
+    - [Binaries](#binaries)
+    - [Packages](#packages)
+  - [Building](#building)
+  - [Configuration](#configuration)
+    - [Basic configuration](#basic-configuration)
+    - [Settings](#settings)
+    - [Advanced configuration](#advanced-configuration)
+    - [Examples](#examples)
+      - [Bridge mattermost (off-topic) - irc (#testing)](#bridge-mattermost-off-topic---irc-testing)
+      - [Bridge slack (#general) - discord (general)](#bridge-slack-general---discord-general)
+  - [Running](#running)
+    - [Docker](#docker)
+  - [Changelog](#changelog)
+  - [FAQ](#faq)
+  - [Related projects](#related-projects)
+  - [Articles](#articles)
+  - [Thanks](#thanks)
 
 ## Features
-* [Support bridging between any protocols](https://github.com/42wim/matterbridge/wiki/Features#support-bridging-between-any-protocols)
-* [Support multiple gateways(bridges) for your protocols](https://github.com/42wim/matterbridge/wiki/Features#support-multiple-gatewaysbridges-for-your-protocols)
-* [Message edits and deletes](https://github.com/42wim/matterbridge/wiki/Features#message-edits-and-deletes)
-* Preserves threading when possible
-* [Attachment / files handling](https://github.com/42wim/matterbridge/wiki/Features#attachment--files-handling)
-* [Username and avatar spoofing](https://github.com/42wim/matterbridge/wiki/Features#username-and-avatar-spoofing)
-* [Private groups](https://github.com/42wim/matterbridge/wiki/Features#private-groups)
-* [API](https://github.com/42wim/matterbridge/wiki/Features#api)
+
+- [Support bridging between any protocols](https://github.com/42wim/matterbridge/wiki/Features#support-bridging-between-any-protocols)
+- [Support multiple gateways(bridges) for your protocols](https://github.com/42wim/matterbridge/wiki/Features#support-multiple-gatewaysbridges-for-your-protocols)
+- [Message edits and deletes](https://github.com/42wim/matterbridge/wiki/Features#message-edits-and-deletes)
+- Preserves threading when possible
+- [Attachment / files handling](https://github.com/42wim/matterbridge/wiki/Features#attachment--files-handling)
+- [Username and avatar spoofing](https://github.com/42wim/matterbridge/wiki/Features#username-and-avatar-spoofing)
+- [Private groups](https://github.com/42wim/matterbridge/wiki/Features#private-groups)
+- [API](https://github.com/42wim/matterbridge/wiki/Features#api)
 
 ### Natively supported
 
-* [Mattermost](https://github.com/mattermost/mattermost-server/) 4.x, 5.x
-* [IRC](http://www.mirc.com/servers.html)
-* [XMPP](https://xmpp.org)
-* [Gitter](https://gitter.im)
-* [Slack](https://slack.com)
-* [Discord](https://discordapp.com)
-* [Telegram](https://telegram.org)
-* [Hipchat](https://www.hipchat.com)
-* [Rocket.chat](https://rocket.chat)
-* [Matrix](https://matrix.org)
-* [Steam](https://store.steampowered.com/)
-* [Twitch](https://twitch.tv)
-* [Ssh-chat](https://github.com/shazow/ssh-chat)
-* [WhatsApp](https://www.whatsapp.com/)
-* [Zulip](https://zulipchat.com)
+- [Discord](https://discordapp.com)
+- [Gitter](https://gitter.im)
+- [IRC](http://www.mirc.com/servers.html)
+- [Keybase](https://keybase.io)
+- [Matrix](https://matrix.org)
+- [Mattermost](https://github.com/mattermost/mattermost-server/) 4.x, 5.x
+- [Microsoft Teams](https://teams.microsoft.com)
+- [Nextcloud Talk](https://nextcloud.com/talk/)
+- [Rocket.chat](https://rocket.chat)
+- [Slack](https://slack.com)
+- [Ssh-chat](https://github.com/shazow/ssh-chat)
+- [Steam](https://store.steampowered.com/)
+- [Telegram](https://telegram.org)
+- [Twitch](https://twitch.tv)
+- [WhatsApp](https://www.whatsapp.com/)
+- [XMPP](https://xmpp.org)
+- [Zulip](https://zulipchat.com)
 
 ### 3rd party via matterbridge api
-* [Minecraft](https://github.com/elytra/MatterLink)
-* [Reddit](https://github.com/bonehurtingjuice/mattereddit)
-* [Facebook messenger](https://github.com/VictorNine/fbridge)
-* [Discourse](https://github.com/DeclanHoare/matterbabble)
+
+- [Discourse](https://github.com/DeclanHoare/matterbabble)
+- [Facebook messenger](https://github.com/VictorNine/fbridge)
+- [Minecraft](https://github.com/elytra/MatterLink)
+- [Reddit](https://github.com/bonehurtingjuice/mattereddit)
+- [Counter-Strike, half-life and more](https://forums.alliedmods.net/showthread.php?t=319430)
+- [MatterAMXX](https://github.com/GabeIggy/MatterAMXX)
 
 ### API
-The API is very basic at the moment.   
+
+The API is basic at the moment.
 More info and examples on the [wiki](https://github.com/42wim/matterbridge/wiki/Api).
 
 Used by the projects below. Feel free to make a PR to add your project to this list.
 
-* [MatterLink](https://github.com/elytra/MatterLink) (Matterbridge link for Minecraft Server chat)
-* [pyCord](https://github.com/NikkyAI/pyCord) (crossplatform chatbot)
-* [Mattereddit](https://github.com/bonehurtingjuice/mattereddit) (Reddit chat support)
-* [fbridge](https://github.com/VictorNine/fbridge) (Facebook messenger support)
-* [matterbabble](https://github.com/DeclanHoare/matterbabble) (Discourse support)
+- [MatterLink](https://github.com/elytra/MatterLink) (Matterbridge link for Minecraft Server chat)
+- [pyCord](https://github.com/NikkyAI/pyCord) (crossplatform chatbot)
+- [Mattereddit](https://github.com/bonehurtingjuice/mattereddit) (Reddit chat support)
+- [fbridge](https://github.com/VictorNine/fbridge) (Facebook messenger support)
+- [matterbabble](https://github.com/DeclanHoare/matterbabble) (Discourse support)
+- [MatterAMXX](https://forums.alliedmods.net/showthread.php?t=319430) (Counter-Strike, half-life and more via AMXX mod)
 
 ## Chat with us
 
 Questions or want to test on your favorite platform? Join below:
 
-* [Gitter][mb-gitter]
-* [IRC][mb-irc]
-* [Discord][mb-discord]
-* [Matrix][mb-matrix]
-* [Slack][mb-slack]
-* [Mattermost][mb-mattermost]
-* [Rocket.Chat][mb-rocketchat]
-* [XMPP][mb-xmpp]
-* [Twitch][mb-twitch]
-* [Zulip][mb-zulip]
-* [Telegram][mb-telegram]
+- [Discord][mb-discord]
+- [Gitter][mb-gitter]
+- [IRC][mb-irc]
+- [Keybase][mb-keybase]
+- [Matrix][mb-matrix]
+- [Mattermost][mb-mattermost]
+- [Rocket.Chat][mb-rocketchat]
+- [Slack][mb-slack]
+- [Telegram][mb-telegram]
+- [Twitch][mb-twitch]
+- [XMPP][mb-xmpp] (matterbridge@conference.jabber.de)
+- [Zulip][mb-zulip]
 
 ## Screenshots
+
 See https://github.com/42wim/matterbridge/wiki
 
-## Installing
+## Installing / upgrading
+
 ### Binaries
-* Latest stable release [v1.14.2](https://github.com/42wim/matterbridge/releases/latest)
-* Development releases (follows master) can be downloaded [here](https://dl.bintray.com/42wim/nightly/)
+
+- Latest stable release [v1.18.1](https://github.com/42wim/matterbridge/releases/latest)
+- Development releases (follows master) can be downloaded [here](https://github.com/42wim/matterbridge/actions) selecting the latest green build and then artifacts.
+
+To install or upgrade just download the latest [binary](https://github.com/42wim/matterbridge/releases/latest) and follow the instructions on the [howto](https://github.com/42wim/matterbridge/wiki/How-to-create-your-config) for a step by step walkthrough for creating your configuration.
 
 ### Packages
-* [Overview](https://repology.org/metapackage/matterbridge/versions)
 
-### Building
-Go 1.9+ is required. Make sure you have [Go](https://golang.org/doc/install) properly installed, including setting up your [GOPATH](https://golang.org/doc/code.html#GOPATH).
+- [Overview](https://repology.org/metapackage/matterbridge/versions)
+- [snap](https://snapcraft.io/matterbridge)
+
+## Building
+
+Most people just want to use binaries, you can find those [here](https://github.com/42wim/matterbridge/releases/latest)
+
+If you really want to build from source, follow these instructions:
+Go 1.12+ is required. Make sure you have [Go](https://golang.org/doc/install) properly installed.
 
-After Go is setup, download matterbridge to your $GOPATH directory.
 
 ```
-cd $GOPATH
 go get github.com/42wim/matterbridge
 ```
 
-You should now have matterbridge binary in the bin directory:
+You should now have matterbridge binary in the ~/go/bin directory:
 
 ```
-$ ls bin/
+$ ls ~/go/bin/
 matterbridge
 ```
 
 ## Configuration
+
 ### Basic configuration
+
 See [howto](https://github.com/42wim/matterbridge/wiki/How-to-create-your-config) for a step by step walkthrough for creating your configuration.
 
+### Settings
+
+All possible [settings](https://github.com/42wim/matterbridge/wiki/Settings) for each bridge.
+
 ### Advanced configuration
-* [matterbridge.toml.sample](https://github.com/42wim/matterbridge/blob/master/matterbridge.toml.sample) for documentation and an example.
+
+- [matterbridge.toml.sample](https://github.com/42wim/matterbridge/blob/master/matterbridge.toml.sample) for documentation and an example.
 
 ### Examples
+
 #### Bridge mattermost (off-topic) - irc (#testing)
+
 ```toml
 [irc]
     [irc.freenode]
@@ -189,6 +227,7 @@ enable=true
 ```
 
 #### Bridge slack (#general) - discord (general)
+
 ```toml
 [slack]
 [slack.test]
@@ -233,12 +272,11 @@ Usage of ./matterbridge:
 ```
 
 ### Docker
-Create your matterbridge.toml file locally eg in `/tmp/matterbridge.toml`
-```
-docker run -ti -v /tmp/matterbridge.toml:/matterbridge.toml 42wim/matterbridge
-```
+
+Please take a look at the [Docker Wiki page](https://github.com/42wim/matterbridge/wiki/Deploy:-Docker) for more information.
 
 ## Changelog
+
 See [changelog.md](https://github.com/42wim/matterbridge/blob/master/changelog.md)
 
 ## FAQ
@@ -246,28 +284,31 @@ See [changelog.md](https://github.com/42wim/matterbridge/blob/master/changelog.m
 See [FAQ](https://github.com/42wim/matterbridge/wiki/FAQ)
 
 ## Related projects
-* [FOSSRIT/infrastructure - roles/matterbridge](https://github.com/FOSSRIT/infrastructure/tree/master/roles/matterbridge) (Ansible role used to automate deployments of Matterbridge)
-* [matterbridge autoconfig](https://github.com/patcon/matterbridge-autoconfig)
-* [matterbridge config viewer](https://github.com/patcon/matterbridge-heroku-viewer)
-* [matterbridge-heroku](https://github.com/cadecairos/matterbridge-heroku)
-* [mattereddit](https://github.com/bonehurtingjuice/mattereddit)
-* [matterlink](https://github.com/elytra/MatterLink)
-* [mattermost-plugin](https://github.com/matterbridge/mattermost-plugin) - Run matterbridge as a plugin in mattermost
-* [pyCord](https://github.com/NikkyAI/pyCord) (crossplatform chatbot)
-* [fbridge](https://github.com/VictorNine/fbridge) (Facebook messenger support)
-* [isla](https://github.com/alphachung/isla) (Bot for Discord-Telegram groups used alongside matterbridge)
-* [matterbabble](https://github.com/DeclanHoare/matterbabble) (Connect Discourse threads to Matterbridge)
+
+- [jwflory/ansible-role-matterbridge](https://galaxy.ansible.com/jwflory/matterbridge) (Ansible role to simplify deploying Matterbridge)
+- [matterbridge autoconfig](https://github.com/patcon/matterbridge-autoconfig)
+- [matterbridge config viewer](https://github.com/patcon/matterbridge-heroku-viewer)
+- [matterbridge-heroku](https://github.com/cadecairos/matterbridge-heroku)
+- [mattereddit](https://github.com/bonehurtingjuice/mattereddit)
+- [matterlink](https://github.com/elytra/MatterLink)
+- [mattermost-plugin](https://github.com/matterbridge/mattermost-plugin) - Run matterbridge as a plugin in mattermost
+- [pyCord](https://github.com/NikkyAI/pyCord) (crossplatform chatbot)
+- [fbridge](https://github.com/VictorNine/fbridge) (Facebook messenger support)
+- [isla](https://github.com/alphachung/isla) (Bot for Discord-Telegram groups used alongside matterbridge)
+- [matterbabble](https://github.com/DeclanHoare/matterbabble) (Connect Discourse threads to Matterbridge)
 
 ## Articles
-* [matterbridge on kubernetes](https://medium.freecodecamp.org/using-kubernetes-to-deploy-a-chat-gateway-or-when-technology-works-like-its-supposed-to-a169a8cd69a3)
-* https://mattermost.com/blog/connect-irc-to-mattermost/
-* https://blog.valvin.fr/2016/09/17/mattermost-et-un-channel-irc-cest-possible/
-* https://blog.brightscout.com/top-10-mattermost-integrations/
-* http://bencey.co.nz/2018/09/17/bridge/
-* https://www.algoo.fr/blog/2018/01/19/recouvrez-votre-liberte-en-quittant-slack-pour-un-mattermost-auto-heberge/
-* https://kopano.com/blog/matterbridge-bridging-mattermost-chat/
-* https://www.stitcher.com/s/?eid=52382713
-* https://daniele.tech/2019/02/how-to-use-matterbridge-to-connect-2-different-slack-workspaces/
+
+- [matterbridge on kubernetes](https://medium.freecodecamp.org/using-kubernetes-to-deploy-a-chat-gateway-or-when-technology-works-like-its-supposed-to-a169a8cd69a3)
+- https://mattermost.com/blog/connect-irc-to-mattermost/
+- https://blog.valvin.fr/2016/09/17/mattermost-et-un-channel-irc-cest-possible/
+- https://blog.brightscout.com/top-10-mattermost-integrations/
+- http://bencey.co.nz/2018/09/17/bridge/
+- https://www.algoo.fr/blog/2018/01/19/recouvrez-votre-liberte-en-quittant-slack-pour-un-mattermost-auto-heberge/
+- https://kopano.com/blog/matterbridge-bridging-mattermost-chat/
+- https://www.stitcher.com/s/?eid=52382713
+- https://daniele.tech/2019/02/how-to-use-matterbridge-to-connect-2-different-slack-workspaces/
+- https://userlinux.net/mattermost-and-matterbridge.html
 
 ## Thanks
 
@@ -279,34 +320,39 @@ See [FAQ](https://github.com/42wim/matterbridge/wiki/FAQ)
 </p>
 
 Matterbridge wouldn't exist without these libraries:
-* discord - https://github.com/bwmarrin/discordgo
-* echo - https://github.com/labstack/echo
-* gitter - https://github.com/sromku/go-gitter
-* gops - https://github.com/google/gops
-* gozulipbot - https://github.com/ifo/gozulipbot
-* irc - https://github.com/lrstanley/girc
-* mattermost - https://github.com/mattermost/mattermost-server
-* matrix - https://github.com/matrix-org/gomatrix
-* sshchat - https://github.com/shazow/ssh-chat
-* slack - https://github.com/nlopes/slack
-* steam - https://github.com/Philipp15b/go-steam
-* telegram - https://github.com/go-telegram-bot-api/telegram-bot-api
-* xmpp - https://github.com/mattn/go-xmpp
-* whatsapp - https://github.com/Rhymen/go-whatsapp/
-* zulip - https://github.com/ifo/gozulipbot
-* tengo - https://github.com/d5/tengo
+
+- discord - https://github.com/bwmarrin/discordgo
+- echo - https://github.com/labstack/echo
+- gitter - https://github.com/sromku/go-gitter
+- gops - https://github.com/google/gops
+- gozulipbot - https://github.com/ifo/gozulipbot
+- irc - https://github.com/lrstanley/girc
+- keybase - https://github.com/keybase/go-keybase-chat-bot
+- matrix - https://github.com/matrix-org/gomatrix
+- mattermost - https://github.com/mattermost/mattermost-server
+- msgraph.go - https://github.com/yaegashi/msgraph.go
+- slack - https://github.com/nlopes/slack
+- sshchat - https://github.com/shazow/ssh-chat
+- steam - https://github.com/Philipp15b/go-steam
+- telegram - https://github.com/go-telegram-bot-api/telegram-bot-api
+- tengo - https://github.com/d5/tengo
+- whatsapp - https://github.com/Rhymen/go-whatsapp/
+- xmpp - https://github.com/mattn/go-xmpp
+- zulip - https://github.com/ifo/gozulipbot
 
 <!-- Links -->
 
-   [mb-gitter]: https://gitter.im/42wim/matterbridge
-   [mb-irc]: https://webchat.freenode.net/?channels=matterbridgechat
-   [mb-discord]: https://discord.gg/AkKPtrQ
-   [mb-matrix]: https://riot.im/app/#/room/#matterbridge:matrix.org
-   [mb-slack]: https://join.slack.com/matterbridgechat/shared_invite/MjEwODMxNjU1NDMwLTE0OTk2MTU3NTMtMzZkZmRiNDZhOA
-   [mb-mattermost]: https://framateam.org/signup_user_complete/?id=tfqm33ggop8x3qgu4boeieta6e
-   [mb-rocketchat]: https://open.rocket.chat/channel/matterbridge
-   [mb-xmpp]: https://inverse.chat/
-   [mb-twitch]: https://www.twitch.tv/matterbridge
-   [mb-whatsapp]: https://www.whatsapp.com/
-   [mb-zulip]: https://matterbridge.zulipchat.com/register/
-   [mb-telegram]: https://t.me/Matterbridge
+[mb-discord]: https://discord.gg/AkKPtrQ
+[mb-gitter]: https://gitter.im/42wim/matterbridge
+[mb-irc]: https://webchat.freenode.net/?channels=matterbridgechat
+[mb-keybase]: https://keybase.io/team/matterbridge
+[mb-matrix]: https://riot.im/app/#/room/#matterbridge:matrix.org
+[mb-mattermost]: https://framateam.org/signup_user_complete/?id=tfqm33ggop8x3qgu4boeieta6e
+[mb-msteams]: https://teams.microsoft.com/join/hj92x75gd3y7
+[mb-rocketchat]: https://open.rocket.chat/channel/matterbridge
+[mb-slack]: https://join.slack.com/matterbridgechat/shared_invite/MjEwODMxNjU1NDMwLTE0OTk2MTU3NTMtMzZkZmRiNDZhOA
+[mb-telegram]: https://t.me/Matterbridge
+[mb-twitch]: https://www.twitch.tv/matterbridge
+[mb-whatsapp]: https://www.whatsapp.com/
+[mb-xmpp]: https://inverse.chat/
+[mb-zulip]: https://matterbridge.zulipchat.com/register/

bridge/api/api.go

@@ -8,9 +8,10 @@ import (
 
 	"github.com/42wim/matterbridge/bridge"
 	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/gorilla/websocket"
 	"github.com/labstack/echo/v4"
 	"github.com/labstack/echo/v4/middleware"
-	"github.com/zfjagann/golang-ring"
+	ring "github.com/zfjagann/golang-ring"
 )
 
 type API struct {
@@ -41,9 +42,17 @@ func New(cfg *bridge.Config) bridge.Bridger {
 			return key == b.GetString("Token"), nil
 		}))
 	}
+
+	// Set RemoteNickFormat to a sane default
+	if !b.IsKeySet("RemoteNickFormat") {
+		b.Log.Debugln("RemoteNickFormat is unset, defaulting to \"{NICK}\"")
+		b.Config.Config.Viper().Set(b.GetConfigKey("RemoteNickFormat"), "{NICK}")
+	}
+
 	e.GET("/api/health", b.handleHealthcheck)
 	e.GET("/api/messages", b.handleMessages)
 	e.GET("/api/stream", b.handleStream)
+	e.GET("/api/websocket", b.handleWebsocket)
 	e.POST("/api/message", b.handlePostMessage)
 	go func() {
 		if b.GetString("BindAddress") == "" {
@@ -106,13 +115,17 @@ func (b *API) handleMessages(c echo.Context) error {
 	return nil
 }
 
-func (b *API) handleStream(c echo.Context) error {
-	c.Response().Header().Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
-	c.Response().WriteHeader(http.StatusOK)
-	greet := config.Message{
+func (b *API) getGreeting() config.Message {
+	return config.Message{
 		Event:     config.EventAPIConnected,
 		Timestamp: time.Now(),
 	}
+}
+
+func (b *API) handleStream(c echo.Context) error {
+	c.Response().Header().Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
+	c.Response().WriteHeader(http.StatusOK)
+	greet := b.getGreeting()
 	if err := json.NewEncoder(c.Response()).Encode(greet); err != nil {
 		return err
 	}
@@ -128,3 +141,52 @@ func (b *API) handleStream(c echo.Context) error {
 		time.Sleep(200 * time.Millisecond)
 	}
 }
+
+func (b *API) handleWebsocketMessage(message config.Message) {
+	message.Channel = "api"
+	message.Protocol = "api"
+	message.Account = b.Account
+	message.ID = ""
+	message.Timestamp = time.Now()
+
+	b.Log.Debugf("Sending websocket message from %s on %s to gateway", message.Username, "api")
+	b.Remote <- message
+}
+
+func (b *API) writePump(conn *websocket.Conn) {
+	for {
+		msg := b.Messages.Dequeue()
+		if msg != nil {
+			err := conn.WriteJSON(msg)
+			if err != nil {
+				break
+			}
+		}
+	}
+}
+
+func (b *API) readPump(conn *websocket.Conn) {
+	for {
+		message := config.Message{}
+		err := conn.ReadJSON(&message)
+		if err != nil {
+			break
+		}
+		b.handleWebsocketMessage(message)
+	}
+}
+
+func (b *API) handleWebsocket(c echo.Context) error {
+	conn, err := websocket.Upgrade(c.Response().Writer, c.Request(), nil, 1024, 1024)
+	if err != nil {
+		return err
+	}
+
+	greet := b.getGreeting()
+	_ = conn.WriteJSON(greet)
+
+	go b.writePump(conn)
+	go b.readPump(conn)
+
+	return nil
+}

bridge/bridge.go

@@ -1,8 +1,10 @@
 package bridge
 
 import (
+	"log"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/sirupsen/logrus"
@@ -41,6 +43,10 @@ type Factory func(*Config) Bridger
 
 func New(bridge *config.Bridge) *Bridge {
 	accInfo := strings.Split(bridge.Account, ".")
+	if len(accInfo) != 2 {
+		log.Fatalf("config failure, account incorrect: %s", bridge.Account)
+	}
+
 	protocol := accInfo[0]
 	name := accInfo[1]
 
@@ -69,6 +75,7 @@ func (b *Bridge) joinChannels(channels map[string]config.ChannelInfo, exists map
 	for ID, channel := range channels {
 		if !exists[ID] {
 			b.Log.Infof("%s: joining %s (ID: %s)", b.Account, channel.Name, ID)
+			time.Sleep(time.Duration(b.GetInt("JoinDelay")) * time.Millisecond)
 			err := b.JoinChannel(channel)
 			if err != nil {
 				return err
@@ -79,8 +86,16 @@ func (b *Bridge) joinChannels(channels map[string]config.ChannelInfo, exists map
 	return nil
 }
 
+func (b *Bridge) GetConfigKey(key string) string {
+	return b.Account + "." + key
+}
+
+func (b *Bridge) IsKeySet(key string) bool {
+	return b.Config.IsKeySet(b.GetConfigKey(key)) || b.Config.IsKeySet("general."+key)
+}
+
 func (b *Bridge) GetBool(key string) bool {
-	val, ok := b.Config.GetBool(b.Account + "." + key)
+	val, ok := b.Config.GetBool(b.GetConfigKey(key))
 	if !ok {
 		val, _ = b.Config.GetBool("general." + key)
 	}
@@ -88,7 +103,7 @@ func (b *Bridge) GetBool(key string) bool {
 }
 
 func (b *Bridge) GetInt(key string) int {
-	val, ok := b.Config.GetInt(b.Account + "." + key)
+	val, ok := b.Config.GetInt(b.GetConfigKey(key))
 	if !ok {
 		val, _ = b.Config.GetInt("general." + key)
 	}
@@ -96,7 +111,7 @@ func (b *Bridge) GetInt(key string) int {
 }
 
 func (b *Bridge) GetString(key string) string {
-	val, ok := b.Config.GetString(b.Account + "." + key)
+	val, ok := b.Config.GetString(b.GetConfigKey(key))
 	if !ok {
 		val, _ = b.Config.GetString("general." + key)
 	}
@@ -104,7 +119,7 @@ func (b *Bridge) GetString(key string) string {
 }
 
 func (b *Bridge) GetStringSlice(key string) []string {
-	val, ok := b.Config.GetStringSlice(b.Account + "." + key)
+	val, ok := b.Config.GetStringSlice(b.GetConfigKey(key))
 	if !ok {
 		val, _ = b.Config.GetStringSlice("general." + key)
 	}
@@ -112,7 +127,7 @@ func (b *Bridge) GetStringSlice(key string) []string {
 }
 
 func (b *Bridge) GetStringSlice2D(key string) [][]string {
-	val, ok := b.Config.GetStringSlice2D(b.Account + "." + key)
+	val, ok := b.Config.GetStringSlice2D(b.GetConfigKey(key))
 	if !ok {
 		val, _ = b.Config.GetStringSlice2D("general." + key)
 	}

bridge/config/config.go

@@ -3,6 +3,8 @@ package config
 import (
 	"bytes"
 	"io/ioutil"
+	"os"
+	"path/filepath"
 	"strings"
 	"sync"
 	"time"
@@ -76,23 +78,29 @@ type Protocol struct {
 	BindAddress            string // mattermost, slack // DEPRECATED
 	Buffer                 int    // api
 	Charset                string // irc
+	ClientID               string // msteams
 	ColorNicks             bool   // only irc for now
 	Debug                  bool   // general
 	DebugLevel             int    // only for irc now
+	DisableWebPagePreview  bool   // telegram
 	EditSuffix             string // mattermost, slack, discord, telegram, gitter
 	EditDisable            bool   // mattermost, slack, discord, telegram, gitter
+	HTMLDisable            bool   // matrix
 	IconURL                string // mattermost, slack
 	IgnoreFailureOnStart   bool   // general
 	IgnoreNicks            string // all protocols
 	IgnoreMessages         string // all protocols
 	Jid                    string // xmpp
+	JoinDelay              string // all protocols
 	Label                  string // all protocols
 	Login                  string // mattermost, matrix
+	LogFile                string // general
 	MediaDownloadBlackList []string
 	MediaDownloadPath      string // Basically MediaServerUpload, but instead of uploading it, just write it to a file on the same server.
 	MediaDownloadSize      int    // all protocols
 	MediaServerDownload    string
 	MediaServerUpload      string
+	MediaConvertTgs        string     // telegram
 	MediaConvertWebPToPNG  bool       // telegram
 	MessageDelay           int        // IRC, time in millisecond to wait between messages
 	MessageFormat          string     // telegram
@@ -116,31 +124,39 @@ type Protocol struct {
 	Protocol               string     // all protocols
 	QuoteDisable           bool       // telegram
 	QuoteFormat            string     // telegram
+	QuoteLengthLimit       int        // telegram
 	RejoinDelay            int        // IRC
 	ReplaceMessages        [][]string // all protocols
 	ReplaceNicks           [][]string // all protocols
 	RemoteNickFormat       string     // all protocols
-	RunCommands            []string   // irc
+	RunCommands            []string   // IRC
 	Server                 string     // IRC,mattermost,XMPP,discord
+	SessionFile            string     // msteams,whatsapp
 	ShowJoinPart           bool       // all protocols
 	ShowTopicChange        bool       // slack
 	ShowUserTyping         bool       // slack
 	ShowEmbeds             bool       // discord
 	SkipTLSVerify          bool       // IRC, mattermost
+	SkipVersionCheck       bool       // mattermost
 	StripNick              bool       // all protocols
+	StripMarkdown          bool       // irc
 	SyncTopic              bool       // slack
 	TengoModifyMessage     string     // general
-	Team                   string     // mattermost
+	Team                   string     // mattermost, keybase
+	TeamID                 string     // msteams
+	TenantID               string     // msteams
 	Token                  string     // gitter, slack, discord, api
 	Topic                  string     // zulip
 	URL                    string     // mattermost, slack // DEPRECATED
 	UseAPI                 bool       // mattermost, slack
+	UseLocalAvatar         []string   // discord
 	UseSASL                bool       // IRC
 	UseTLS                 bool       // IRC
 	UseDiscriminator       bool       // discord
 	UseFirstName           bool       // telegram
 	UseUserName            bool       // discord
 	UseInsecureURL         bool       // telegram
+	VerboseJoinPart        bool       // IRC
 	WebhookBindAddress     string     // mattermost, slack
 	WebhookURL             string     // mattermost, slack
 }
@@ -166,6 +182,13 @@ type Gateway struct {
 	InOut  []Bridge
 }
 
+type Tengo struct {
+	InMessage        string
+	Message          string
+	RemoteNickFormat string
+	OutMessage       string
+}
+
 type SameChannelGateway struct {
 	Name     string
 	Enable   bool
@@ -189,13 +212,17 @@ type BridgeValues struct {
 	SSHChat            map[string]Protocol
 	WhatsApp           map[string]Protocol // TODO is this struct used? Search for "SlackLegacy" for example didn't return any results
 	Zulip              map[string]Protocol
+	Keybase            map[string]Protocol
 	General            Protocol
+	Tengo              Tengo
 	Gateway            []Gateway
 	SameChannelGateway []SameChannelGateway
 }
 
 type Config interface {
+	Viper() *viper.Viper
 	BridgeValues() *BridgeValues
+	IsKeySet(key string) bool
 	GetBool(key string) (bool, bool)
 	GetInt(key string) (int, bool)
 	GetString(key string) (string, bool)
@@ -221,7 +248,17 @@ func NewConfig(rootLogger *logrus.Logger, cfgfile string) Config {
 		logger.Fatalf("Failed to read configuration file: %#v", err)
 	}
 
-	mycfg := newConfigFromString(logger, input)
+	cfgtype := detectConfigType(cfgfile)
+	mycfg := newConfigFromString(logger, input, cfgtype)
+	if mycfg.cv.General.LogFile != "" {
+		logfile, err := os.OpenFile(mycfg.cv.General.LogFile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600)
+		if err == nil {
+			logger.Info("Opening log file ", mycfg.cv.General.LogFile)
+			rootLogger.Out = logfile
+		} else {
+			logger.Warn("Failed to open ", mycfg.cv.General.LogFile)
+		}
+	}
 	if mycfg.cv.General.MediaDownloadSize == 0 {
 		mycfg.cv.General.MediaDownloadSize = 1000000
 	}
@@ -232,25 +269,37 @@ func NewConfig(rootLogger *logrus.Logger, cfgfile string) Config {
 	return mycfg
 }
 
+// detectConfigType detects JSON and YAML formats, defaults to TOML.
+func detectConfigType(cfgfile string) string {
+	fileExt := filepath.Ext(cfgfile)
+	switch fileExt {
+	case ".json":
+		return "json"
+	case ".yaml", ".yml":
+		return "yaml"
+	}
+	return "toml"
+}
+
 // NewConfigFromString instantiates a new configuration based on the specified string.
 func NewConfigFromString(rootLogger *logrus.Logger, input []byte) Config {
 	logger := rootLogger.WithFields(logrus.Fields{"prefix": "config"})
-	return newConfigFromString(logger, input)
+	return newConfigFromString(logger, input, "toml")
 }
 
-func newConfigFromString(logger *logrus.Entry, input []byte) *config {
-	viper.SetConfigType("toml")
+func newConfigFromString(logger *logrus.Entry, input []byte, cfgtype string) *config {
+	viper.SetConfigType(cfgtype)
 	viper.SetEnvPrefix("matterbridge")
 	viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_", "-", "_"))
 	viper.AutomaticEnv()
 
 	if err := viper.ReadConfig(bytes.NewBuffer(input)); err != nil {
-		logger.Fatalf("Failed to parse the configuration: %#v", err)
+		logger.Fatalf("Failed to parse the configuration: %s", err)
 	}
 
 	cfg := &BridgeValues{}
 	if err := viper.Unmarshal(cfg); err != nil {
-		logger.Fatalf("Failed to load the configuration: %#v", err)
+		logger.Fatalf("Failed to load the configuration: %s", err)
 	}
 	return &config{
 		logger: logger,
@@ -263,6 +312,16 @@ func (c *config) BridgeValues() *BridgeValues {
 	return c.cv
 }
 
+func (c *config) Viper() *viper.Viper {
+	return c.v
+}
+
+func (c *config) IsKeySet(key string) bool {
+	c.RLock()
+	defer c.RUnlock()
+	return c.v.IsSet(key)
+}
+
 func (c *config) GetBool(key string) (bool, bool) {
 	c.RLock()
 	defer c.RUnlock()
@@ -322,6 +381,11 @@ type TestConfig struct {
 	Overrides map[string]interface{}
 }
 
+func (c *TestConfig) IsKeySet(key string) bool {
+	_, ok := c.Overrides[key]
+	return ok || c.Config.IsKeySet(key)
+}
+
 func (c *TestConfig) GetBool(key string) (bool, bool) {
 	val, ok := c.Overrides[key]
 	if ok {

bridge/discord/discord.go

@@ -10,7 +10,7 @@ import (
 	"github.com/42wim/matterbridge/bridge"
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/42wim/matterbridge/bridge/helper"
-	"github.com/bwmarrin/discordgo"
+	"github.com/matterbridge/discordgo"
 )
 
 const MessageLength = 1950
@@ -21,7 +21,7 @@ type Bdiscord struct {
 	c *discordgo.Session
 
 	nick            string
-	useChannelID    bool
+	userID          string
 	guildID         string
 	webhookID       string
 	webhookToken    string
@@ -34,6 +34,8 @@ type Bdiscord struct {
 	membersMutex  sync.RWMutex
 	userMemberMap map[string]*discordgo.Member
 	nickMemberMap map[string]*discordgo.Member
+	webhookCache  map[string]string
+	webhookMutex  sync.RWMutex
 }
 
 func New(cfg *bridge.Config) bridge.Bridger {
@@ -41,6 +43,7 @@ func New(cfg *bridge.Config) bridge.Bridger {
 	b.userMemberMap = make(map[string]*discordgo.Member)
 	b.nickMemberMap = make(map[string]*discordgo.Member)
 	b.channelInfoMap = make(map[string]*config.ChannelInfo)
+	b.webhookCache = make(map[string]string)
 	if b.GetString("WebhookURL") != "" {
 		b.Log.Debug("Configuring Discord Incoming Webhook")
 		b.webhookID, b.webhookToken = b.splitURL(b.GetString("WebhookURL"))
@@ -72,9 +75,11 @@ func (b *Bdiscord) Connect() error {
 	}
 	b.Log.Info("Connection succeeded")
 	b.c.AddHandler(b.messageCreate)
+	b.c.AddHandler(b.messageTyping)
 	b.c.AddHandler(b.memberUpdate)
 	b.c.AddHandler(b.messageUpdate)
 	b.c.AddHandler(b.messageDelete)
+	b.c.AddHandler(b.messageDeleteBulk)
 	b.c.AddHandler(b.memberAdd)
 	b.c.AddHandler(b.memberRemove)
 	err = b.c.Open()
@@ -91,15 +96,16 @@ func (b *Bdiscord) Connect() error {
 	}
 	serverName := strings.Replace(b.GetString("Server"), "ID:", "", -1)
 	b.nick = userinfo.Username
+	b.userID = userinfo.ID
 	b.channelsMutex.Lock()
 	for _, guild := range guilds {
 		if guild.Name == serverName || guild.ID == serverName {
 			b.channels, err = b.c.GuildChannels(guild.ID)
-			b.guildID = guild.ID
-			guildFound = true
 			if err != nil {
 				break
 			}
+			b.guildID = guild.ID
+			guildFound = true
 		}
 	}
 	b.channelsMutex.Unlock()
@@ -113,30 +119,37 @@ func (b *Bdiscord) Connect() error {
 			b.Log.Infof("Server=\"%s\" # Server ID", guild.ID)
 		}
 	}
-
 	if err != nil {
 		return err
 	}
+
 	b.channelsMutex.RLock()
 	if b.GetString("WebhookURL") == "" {
 		for _, channel := range b.channels {
 			b.Log.Debugf("found channel %#v", channel)
 		}
 	} else {
-		b.canEditWebhooks = true
-		for _, channel := range b.channels {
-			b.Log.Debugf("found channel %#v; verifying PermissionManageWebhooks", channel)
-			perms, permsErr := b.c.State.UserChannelPermissions(userinfo.ID, channel.ID)
-			manageWebhooks := discordgo.PermissionManageWebhooks
-			if permsErr != nil || perms&manageWebhooks != manageWebhooks {
-				b.Log.Warnf("Can't manage webhooks in channel \"%s\"", channel.Name)
-				b.canEditWebhooks = false
+		manageWebhooks := discordgo.PermissionManageWebhooks
+		var channelsDenied []string
+		for _, info := range b.Channels {
+			id := b.getChannelID(info.Name) // note(qaisjp): this readlocks channelsMutex
+			b.Log.Debugf("Verifying PermissionManageWebhooks for %s with ID %s", info.ID, id)
+
+			perms, permsErr := b.c.UserChannelPermissions(userinfo.ID, id)
+			if permsErr != nil {
+				b.Log.Warnf("Failed to check PermissionManageWebhooks in channel \"%s\": %s", info.Name, permsErr.Error())
+			} else if perms&manageWebhooks == manageWebhooks {
+				continue
 			}
+			channelsDenied = append(channelsDenied, fmt.Sprintf("%#v", info.Name))
 		}
+
+		b.canEditWebhooks = len(channelsDenied) == 0
 		if b.canEditWebhooks {
 			b.Log.Info("Can manage webhooks; will edit channel for global webhook on send")
 		} else {
 			b.Log.Warn("Can't manage webhooks; won't edit channel for global webhook on send")
+			b.Log.Warn("Can't manage webhooks in channels: ", strings.Join(channelsDenied, ", "))
 		}
 	}
 	b.channelsMutex.RUnlock()
@@ -172,21 +185,27 @@ func (b *Bdiscord) JoinChannel(channel config.ChannelInfo) error {
 	defer b.channelsMutex.Unlock()
 
 	b.channelInfoMap[channel.ID] = &channel
-	idcheck := strings.Split(channel.Name, "ID:")
-	if len(idcheck) > 1 {
-		b.useChannelID = true
-	}
 	return nil
 }
 
 func (b *Bdiscord) Send(msg config.Message) (string, error) {
 	b.Log.Debugf("=> Receiving %#v", msg)
 
+	origMsgID := msg.ID
+
 	channelID := b.getChannelID(msg.Channel)
 	if channelID == "" {
 		return "", fmt.Errorf("Could not find channelID for %v", msg.Channel)
 	}
 
+	if msg.Event == config.EventUserTyping {
+		if b.GetBool("ShowUserTyping") {
+			err := b.c.ChannelTyping(channelID)
+			return "", err
+		}
+		return "", nil
+	}
+
 	// Make a action /me of the message
 	if msg.Event == config.EventUserAction {
 		msg.Text = "_" + msg.Text + "_"
@@ -208,26 +227,27 @@ func (b *Bdiscord) Send(msg config.Message) (string, error) {
 	b.channelsMutex.RUnlock()
 
 	// Use webhook to send the message
-	if wID != "" {
+	if wID != "" && msg.Event != config.EventMsgDelete {
 		// skip events
-		if msg.Event != "" && msg.Event != config.EventJoinLeave && msg.Event != config.EventTopicChange {
+		if msg.Event != "" && msg.Event != config.EventUserAction && msg.Event != config.EventJoinLeave && msg.Event != config.EventTopicChange {
 			return "", nil
 		}
-		b.Log.Debugf("Broadcasting using Webhook")
-		for _, f := range msg.Extra["file"] {
-			fi := f.(config.FileInfo)
-			if fi.Comment != "" {
-				msg.Text += fi.Comment + ": "
-			}
-			if fi.URL != "" {
-				msg.Text = fi.URL
-				if fi.Comment != "" {
-					msg.Text = fi.Comment + ": " + fi.URL
-				}
+
+		// If we are editing a message, delete the old message
+		if msg.ID != "" {
+			msg.ID = b.getCacheID(msg.ID)
+			b.Log.Debugf("Deleting edited webhook message")
+			err := b.c.ChannelMessageDelete(channelID, msg.ID)
+			if err != nil {
+				b.Log.Errorf("Could not delete edited webhook message: %s", err)
 			}
 		}
+
+		b.Log.Debugf("Broadcasting using Webhook")
+
 		// skip empty messages
-		if msg.Text == "" {
+		if msg.Text == "" && (msg.Extra == nil || len(msg.Extra["file"]) == 0) {
+			b.Log.Debugf("Skipping empty message %#v", msg)
 			return "", nil
 		}
 
@@ -250,16 +270,18 @@ func (b *Bdiscord) Send(msg config.Message) (string, error) {
 				return "", err
 			}
 		}
-		err := b.c.WebhookExecute(
-			wID,
-			wToken,
-			true,
-			&discordgo.WebhookParams{
-				Content:   msg.Text,
-				Username:  msg.Username,
-				AvatarURL: msg.Avatar,
-			})
-		return "", err
+		b.Log.Debugf("Processing webhook sending for message %#v", msg)
+		msg, err := b.webhookSend(&msg, wID, wToken)
+		if err != nil {
+			b.Log.Errorf("Could not broadcast via webook for message %#v: %s", msg, err)
+			return "", err
+		}
+		if msg == nil {
+			return "", nil
+		}
+
+		b.updateCacheID(origMsgID, msg.ID)
+		return msg.ID, nil
 	}
 
 	b.Log.Debugf("Broadcasting using token (API)")
@@ -269,6 +291,7 @@ func (b *Bdiscord) Send(msg config.Message) (string, error) {
 		if msg.ID == "" {
 			return "", nil
 		}
+		msg.ID = b.getCacheID(msg.ID)
 		err := b.c.ChannelMessageDelete(channelID, msg.ID)
 		return "", err
 	}
@@ -301,7 +324,7 @@ func (b *Bdiscord) Send(msg config.Message) (string, error) {
 	if err != nil {
 		return "", err
 	}
-	return res.ID, err
+	return res.ID, nil
 }
 
 // useWebhook returns true if we have a webhook defined somewhere
@@ -365,3 +388,83 @@ func (b *Bdiscord) handleUploadFile(msg *config.Message, channelID string) (stri
 	}
 	return "", nil
 }
+
+// webhookSend send one or more message via webhook, taking care of file
+// uploads (from slack, telegram or mattermost).
+// Returns messageID and error.
+func (b *Bdiscord) webhookSend(msg *config.Message, webhookID, token string) (*discordgo.Message, error) {
+	var (
+		res *discordgo.Message
+		err error
+	)
+
+	// If avatar is unset, check if UseLocalAvatar contains the message's
+	// account or protocol, and if so, try to find a local avatar
+	if msg.Avatar == "" {
+		for _, val := range b.GetStringSlice("UseLocalAvatar") {
+			if msg.Protocol == val || msg.Account == val {
+				if avatar := b.findAvatar(msg); avatar != "" {
+					msg.Avatar = avatar
+				}
+				break
+			}
+		}
+	}
+
+	// WebhookParams can have either `Content` or `File`.
+
+	// We can't send empty messages.
+	if msg.Text != "" {
+		res, err = b.c.WebhookExecute(
+			webhookID,
+			token,
+			true,
+			&discordgo.WebhookParams{
+				Content:   msg.Text,
+				Username:  msg.Username,
+				AvatarURL: msg.Avatar,
+			},
+		)
+		if err != nil {
+			b.Log.Errorf("Could not send text (%s) for message %#v: %s", msg.Text, msg, err)
+		}
+	}
+
+	if msg.Extra != nil {
+		for _, f := range msg.Extra["file"] {
+			fi := f.(config.FileInfo)
+			file := discordgo.File{
+				Name:        fi.Name,
+				ContentType: "",
+				Reader:      bytes.NewReader(*fi.Data),
+			}
+			content := ""
+			if msg.Text == "" {
+				content = fi.Comment
+			}
+			_, e2 := b.c.WebhookExecute(
+				webhookID,
+				token,
+				false,
+				&discordgo.WebhookParams{
+					Username:  msg.Username,
+					AvatarURL: msg.Avatar,
+					File:      &file,
+					Content:   content,
+				},
+			)
+			if e2 != nil {
+				b.Log.Errorf("Could not send file %#v for message %#v: %s", file, msg, e2)
+			}
+		}
+	}
+	return res, err
+}
+
+func (b *Bdiscord) findAvatar(m *config.Message) string {
+	member, err := b.getGuildMemberByNick(m.Username)
+	if err != nil {
+		return ""
+	}
+	return member.User.AvatarURL("")
+}

bridge/discord/handlers.go

@@ -2,20 +2,50 @@ package bdiscord
 
 import (
 	"github.com/42wim/matterbridge/bridge/config"
-	"github.com/bwmarrin/discordgo"
+	"github.com/matterbridge/discordgo"
 )
 
 func (b *Bdiscord) messageDelete(s *discordgo.Session, m *discordgo.MessageDelete) { //nolint:unparam
 	rmsg := config.Message{Account: b.Account, ID: m.ID, Event: config.EventMsgDelete, Text: config.EventMsgDelete}
 	rmsg.Channel = b.getChannelName(m.ChannelID)
-	if b.useChannelID {
-		rmsg.Channel = "ID:" + m.ChannelID
-	}
+
 	b.Log.Debugf("<= Sending message from %s to gateway", b.Account)
 	b.Log.Debugf("<= Message is %#v", rmsg)
 	b.Remote <- rmsg
 }
 
+// TODO(qaisjp): if other bridges support bulk deletions, it could be fanned out centrally
+func (b *Bdiscord) messageDeleteBulk(s *discordgo.Session, m *discordgo.MessageDeleteBulk) { //nolint:unparam
+	for _, msgID := range m.Messages {
+		rmsg := config.Message{
+			Account: b.Account,
+			ID:      msgID,
+			Event:   config.EventMsgDelete,
+			Text:    config.EventMsgDelete,
+			Channel: b.getChannelName(m.ChannelID),
+		}
+
+		b.Log.Debugf("<= Sending message from %s to gateway", b.Account)
+		b.Log.Debugf("<= Message is %#v", rmsg)
+		b.Remote <- rmsg
+	}
+}
+
+func (b *Bdiscord) messageTyping(s *discordgo.Session, m *discordgo.TypingStart) {
+	if !b.GetBool("ShowUserTyping") {
+		return
+	}
+
+	// Ignore our own typing messages
+	if m.UserID == b.userID {
+		return
+	}
+
+	rmsg := config.Message{Account: b.Account, Event: config.EventUserTyping}
+	rmsg.Channel = b.getChannelName(m.ChannelID)
+	b.Remote <- rmsg
+}
+
 func (b *Bdiscord) messageUpdate(s *discordgo.Session, m *discordgo.MessageUpdate) { //nolint:unparam
 	if b.GetBool("EditDisable") {
 		return
@@ -24,7 +54,10 @@ func (b *Bdiscord) messageUpdate(s *discordgo.Session, m *discordgo.MessageUpdat
 	if m.Message.EditedTimestamp != "" {
 		b.Log.Debugf("Sending edit message")
 		m.Content += b.GetString("EditSuffix")
-		b.messageCreate(s, (*discordgo.MessageCreate)(m))
+		msg := &discordgo.MessageCreate{
+			Message: m.Message,
+		}
+		b.messageCreate(s, msg)
 	}
 }
 
@@ -36,7 +69,7 @@ func (b *Bdiscord) messageCreate(s *discordgo.Session, m *discordgo.MessageCreat
 		return
 	}
 	// if using webhooks, do not relay if it's ours
-	if b.useWebhook() && m.Author.Bot { // && b.isWebhookID(m.Author.ID) {
+	if b.useWebhook() && m.Author.Bot && b.isWebhookID(m.Author.ID) {
 		return
 	}
 
@@ -51,7 +84,6 @@ func (b *Bdiscord) messageCreate(s *discordgo.Session, m *discordgo.MessageCreat
 
 	if m.Content != "" {
 		b.Log.Debugf("== Receiving event %#v", m.Message)
-		m.Message.Content = b.stripCustomoji(m.Message.Content)
 		m.Message.Content = b.replaceChannelMentions(m.Message.Content)
 		rmsg.Text, err = m.ContentWithMoreMentionsReplaced(b.c)
 		if err != nil {
@@ -62,16 +94,13 @@ func (b *Bdiscord) messageCreate(s *discordgo.Session, m *discordgo.MessageCreat
 
 	// set channel name
 	rmsg.Channel = b.getChannelName(m.ChannelID)
-	if b.useChannelID {
-		rmsg.Channel = "ID:" + m.ChannelID
-	}
 
-	// set username
-	if !b.GetBool("UseUserName") {
-		rmsg.Username = b.getNick(m.Author)
+	fromWebhook := m.WebhookID != ""
+	if !fromWebhook && !b.GetBool("UseUserName") {
+		rmsg.Username = b.getNick(m.Author, m.GuildID)
 	} else {
 		rmsg.Username = m.Author.Username
-		if b.GetBool("UseDiscriminator") {
+		if !fromWebhook && b.GetBool("UseDiscriminator") {
 			rmsg.Username += "#" + m.Author.Discriminator
 		}
 	}
@@ -79,7 +108,7 @@ func (b *Bdiscord) messageCreate(s *discordgo.Session, m *discordgo.MessageCreat
 	// if we have embedded content add it to text
 	if b.GetBool("ShowEmbeds") && m.Message.Embeds != nil {
 		for _, embed := range m.Message.Embeds {
-			rmsg.Text = rmsg.Text + "embed: " + embed.Title + " - " + embed.Description + " - " + embed.URL + "\n"
+			rmsg.Text += handleEmbed(embed)
 		}
 	}
 
@@ -95,6 +124,9 @@ func (b *Bdiscord) messageCreate(s *discordgo.Session, m *discordgo.MessageCreat
 		rmsg.Event = config.EventUserAction
 	}
 
+	// Replace emotes
+	rmsg.Text = replaceEmotes(rmsg.Text)
+
 	b.Log.Debugf("<= Sending message from %s on %s to gateway", m.Author.Username, b.Account)
 	b.Log.Debugf("<= Message is %#v", rmsg)
 	b.Remote <- rmsg
@@ -168,3 +200,33 @@ func (b *Bdiscord) memberRemove(s *discordgo.Session, m *discordgo.GuildMemberRe
 	b.Log.Debugf("<= Message is %#v", rmsg)
 	b.Remote <- rmsg
 }
+
+func handleEmbed(embed *discordgo.MessageEmbed) string {
+	var t []string
+	var result string
+
+	t = append(t, embed.Title)
+	t = append(t, embed.Description)
+	t = append(t, embed.URL)
+
+	i := 0
+	for _, e := range t {
+		if e == "" {
+			continue
+		}
+
+		i++
+		if i == 1 {
+			result += " embed: " + e
+			continue
+		}
+
+		result += " - " + e
+	}
+
+	if result != "" {
+		result += "\n"
+	}
+
+	return result
+}

bridge/discord/handlers_test.go

@@ -0,0 +1,58 @@
+package bdiscord
+
+import (
+	"testing"
+
+	"github.com/matterbridge/discordgo"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestHandleEmbed(t *testing.T) {
+	testcases := map[string]struct {
+		embed  *discordgo.MessageEmbed
+		result string
+	}{
+		"allempty": {
+			embed:  &discordgo.MessageEmbed{},
+			result: "",
+		},
+		"one": {
+			embed: &discordgo.MessageEmbed{
+				Title: "blah",
+			},
+			result: " embed: blah\n",
+		},
+		"two": {
+			embed: &discordgo.MessageEmbed{
+				Title:       "blah",
+				Description: "blah2",
+			},
+			result: " embed: blah - blah2\n",
+		},
+		"three": {
+			embed: &discordgo.MessageEmbed{
+				Title:       "blah",
+				Description: "blah2",
+				URL:         "blah3",
+			},
+			result: " embed: blah - blah2 - blah3\n",
+		},
+		"twob": {
+			embed: &discordgo.MessageEmbed{
+				Description: "blah2",
+				URL:         "blah3",
+			},
+			result: " embed: blah2 - blah3\n",
+		},
+		"oneb": {
+			embed: &discordgo.MessageEmbed{
+				URL: "blah3",
+			},
+			result: " embed: blah3\n",
+		},
+	}
+
+	for name, tc := range testcases {
+		assert.Equalf(t, tc.result, handleEmbed(tc.embed), "Testcases %s", name)
+	}
+}

bridge/discord/helpers.go

@@ -6,10 +6,10 @@ import (
 	"strings"
 	"unicode"
 
-	"github.com/bwmarrin/discordgo"
+	"github.com/matterbridge/discordgo"
 )
 
-func (b *Bdiscord) getNick(user *discordgo.User) string {
+func (b *Bdiscord) getNick(user *discordgo.User, guildID string) string {
 	b.membersMutex.RLock()
 	defer b.membersMutex.RUnlock()
 
@@ -23,9 +23,9 @@ func (b *Bdiscord) getNick(user *discordgo.User) string {
 	}
 
 	// If we didn't find nick, search for it.
-	member, err := b.c.GuildMember(b.guildID, user.ID)
+	member, err := b.c.GuildMember(guildID, user.ID)
 	if err != nil {
-		b.Log.Warnf("Failed to fetch information for member %#v: %s", user, err)
+		b.Log.Warnf("Failed to fetch information for member %#v on guild %#v: %s", user, guildID, err)
 		return user.Username
 	} else if member == nil {
 		b.Log.Warnf("Got no information for member %#v", user)
@@ -51,6 +51,9 @@ func (b *Bdiscord) getGuildMemberByNick(nick string) (*discordgo.Member, error)
 }
 
 func (b *Bdiscord) getChannelID(name string) string {
+	if strings.Contains(name, "/") {
+		return b.getCategoryChannelID(name)
+	}
 	b.channelsMutex.RLock()
 	defer b.channelsMutex.RUnlock()
 
@@ -59,40 +62,92 @@ func (b *Bdiscord) getChannelID(name string) string {
 		return idcheck[1]
 	}
 	for _, channel := range b.channels {
-		if channel.Name == name {
+		if channel.Name == name && channel.Type == discordgo.ChannelTypeGuildText {
 			return channel.ID
 		}
 	}
 	return ""
 }
 
+func (b *Bdiscord) getCategoryChannelID(name string) string {
+	b.channelsMutex.RLock()
+	defer b.channelsMutex.RUnlock()
+	res := strings.Split(name, "/")
+	// shouldn't happen because function should be only called from getChannelID
+	if len(res) != 2 {
+		return ""
+	}
+	catName, chanName := res[0], res[1]
+	for _, channel := range b.channels {
+		// if we have a parentID, lookup the name of that parent (category)
+		// and if it matches return it
+		if channel.Name == chanName && channel.ParentID != "" {
+			for _, cat := range b.channels {
+				if cat.ID == channel.ParentID && cat.Name == catName {
+					return channel.ID
+				}
+			}
+		}
+	}
+	return ""
+}
+
 func (b *Bdiscord) getChannelName(id string) string {
 	b.channelsMutex.RLock()
 	defer b.channelsMutex.RUnlock()
 
+	for _, c := range b.channelInfoMap {
+		if c.Name == "ID:"+id {
+			// if we have ID: specified in our gateway configuration return this
+			return c.Name
+		}
+	}
+
 	for _, channel := range b.channels {
 		if channel.ID == id {
-			return channel.Name
+			return b.getCategoryChannelName(channel.Name, channel.ParentID)
 		}
 	}
 	return ""
 }
 
+func (b *Bdiscord) getCategoryChannelName(name, parentID string) string {
+	var usesCat bool
+	// do we have a category configuration in the channel config
+	for _, c := range b.channelInfoMap {
+		if strings.Contains(c.Name, "/") {
+			usesCat = true
+			break
+		}
+	}
+	// configuration without category, return the normal channel name
+	if !usesCat {
+		return name
+	}
+	// create a category/channel response
+	for _, c := range b.channels {
+		if c.ID == parentID {
+			name = c.Name + "/" + name
+		}
+	}
+	return name
+}
+
 var (
 	// See https://discordapp.com/developers/docs/reference#message-formatting.
 	channelMentionRE = regexp.MustCompile("<#[0-9]+>")
-	emojiRE          = regexp.MustCompile("<(:.*?:)[0-9]+>")
 	userMentionRE    = regexp.MustCompile("@[^@\n]{1,32}")
+	emoteRE          = regexp.MustCompile(`<a?(:\w+:)\d+>`)
 )
 
 func (b *Bdiscord) replaceChannelMentions(text string) string {
 	replaceChannelMentionFunc := func(match string) string {
-		var err error
 		channelID := match[2 : len(match)-1]
-
 		channelName := b.getChannelName(channelID)
+
 		// If we don't have the channel refresh our list.
 		if channelName == "" {
+			var err error
 			b.channels, err = b.c.GuildChannels(b.guildID)
 			if err != nil {
 				return "#unknownchannel"
@@ -128,13 +183,14 @@ func (b *Bdiscord) replaceUserMentions(text string) string {
 	return userMentionRE.ReplaceAllStringFunc(text, replaceUserMentionFunc)
 }
 
-func (b *Bdiscord) stripCustomoji(text string) string {
-	return emojiRE.ReplaceAllString(text, `$1`)
+func replaceEmotes(text string) string {
+	return emoteRE.ReplaceAllString(text, "$1")
 }
 
 func (b *Bdiscord) replaceAction(text string) (string, bool) {
-	if strings.HasPrefix(text, "_") && strings.HasSuffix(text, "_") {
-		return text[1:], true
+	length := len(text)
+	if length > 1 && text[0] == '_' && text[length-1] == '_' {
+		return text[1 : length-1], true
 	}
 	return text, false
 }
@@ -153,6 +209,40 @@ func (b *Bdiscord) splitURL(url string) (string, string) {
 	return webhookURLSplit[webhookIdxID], webhookURLSplit[webhookIdxToken]
 }
 
+// getcacheID tries to find a corresponding msgID in the webhook cache.
+// if not found returns the original request.
+func (b *Bdiscord) getCacheID(msgID string) string {
+	b.webhookMutex.RLock()
+	defer b.webhookMutex.RUnlock()
+	for k, v := range b.webhookCache {
+		if msgID == k {
+			return v
+		}
+	}
+	return msgID
+}
+
+// updateCacheID updates the cache so that the newID takes the place of
+// the original ID. This is used for edit/deletes in combination with webhooks
+// as editing a message via webhook means deleting the message and creating a
+// new message (with a new ID). This ID needs to be set instead of the original ID
+func (b *Bdiscord) updateCacheID(origID, newID string) {
+	b.webhookMutex.Lock()
+	match := false
+	for k, v := range b.webhookCache {
+		if v == origID {
+			delete(b.webhookCache, k)
+			b.webhookCache[origID] = newID
+			match = true
+			continue
+		}
+	}
+	if !match && origID != "" {
+		b.webhookCache[origID] = newID
+	}
+	b.webhookMutex.Unlock()
+}
+
 func enumerateUsernames(s string) []string {
 	onlySpace := true
 	for _, r := range s {

bridge/helper/helper.go

@@ -5,7 +5,10 @@ import (
 	"fmt"
 	"image/png"
 	"io"
+	"io/ioutil"
 	"net/http"
+	"os"
+	"os/exec"
 	"regexp"
 	"strings"
 	"time"
@@ -14,8 +17,10 @@ import (
 	"golang.org/x/image/webp"
 
 	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/gomarkdown/markdown"
+	"github.com/gomarkdown/markdown/html"
+	"github.com/gomarkdown/markdown/parser"
 	"github.com/sirupsen/logrus"
-	"gitlab.com/golang-commonmark/markdown"
 )
 
 // DownloadFile downloads the given non-authenticated URL.
@@ -176,15 +181,21 @@ func ClipMessage(text string, length int) string {
 	return text
 }
 
+// ParseMarkdown takes in an input string as markdown and parses it to html
 func ParseMarkdown(input string) string {
-	md := markdown.New(markdown.XHTMLOutput(true), markdown.Breaks(true))
-	res := md.RenderToString([]byte(input))
+	extensions := parser.HardLineBreak | parser.NoIntraEmphasis
+	markdownParser := parser.NewWithExtensions(extensions)
+	renderer := html.NewRenderer(html.RendererOptions{
+		Flags: 0,
+	})
+	parsedMarkdown := markdown.ToHTML([]byte(input), markdownParser, renderer)
+	res := string(parsedMarkdown)
 	res = strings.TrimPrefix(res, "<p>")
 	res = strings.TrimSuffix(res, "</p>\n")
 	return res
 }
 
-// ConvertWebPToPNG convert input data (which should be WebP format to PNG format)
+// ConvertWebPToPNG converts input data (which should be WebP format) to PNG format
 func ConvertWebPToPNG(data *[]byte) error {
 	r := bytes.NewReader(*data)
 	m, err := webp.Decode(r)
@@ -199,3 +210,49 @@ func ConvertWebPToPNG(data *[]byte) error {
 	*data = w.Bytes()
 	return nil
 }
+
+// CanConvertTgsToX Checks whether the external command necessary for ConvertTgsToX works.
+func CanConvertTgsToX() error {
+	// We depend on the fact that `lottie_convert.py --help` has exit status 0.
+	// Hyrum's Law predicted this, and Murphy's Law predicts that this will break eventually.
+	// However, there is no alternative like `lottie_convert.py --is-properly-installed`
+	cmd := exec.Command("lottie_convert.py", "--help")
+	return cmd.Run()
+}
+
+// ConvertTgsToWebP convert input data (which should be tgs format) to WebP format
+// This relies on an external command, which is ugly, but works.
+func ConvertTgsToX(data *[]byte, outputFormat string, logger *logrus.Entry) error {
+	// lottie can't handle input from a pipe, so write to a temporary file:
+	tmpFile, err := ioutil.TempFile(os.TempDir(), "matterbridge-lottie-*.tgs")
+	if err != nil {
+		return err
+	}
+	tmpFileName := tmpFile.Name()
+	defer func() {
+		if removeErr := os.Remove(tmpFileName); removeErr != nil {
+			logger.Errorf("Could not delete temporary file %s: %v", tmpFileName, removeErr)
+		}
+	}()
+
+	if _, writeErr := tmpFile.Write(*data); writeErr != nil {
+		return writeErr
+	}
+	// Must close before calling lottie to avoid data races:
+	if closeErr := tmpFile.Close(); closeErr != nil {
+		return closeErr
+	}
+
+	// Call lottie to transform:
+	cmd := exec.Command("lottie_convert.py", "--input-format", "lottie", "--output-format", outputFormat, tmpFileName, "/dev/stdout")
+	cmd.Stderr = nil
+	// NB: lottie writes progress into to stderr in all cases.
+	stdout, stderr := cmd.Output()
+	if stderr != nil {
+		// 'stderr' already contains some parts of Stderr, because it was set to 'nil'.
+		return stderr
+	}
+
+	*data = stdout
+	return nil
+}

bridge/irc/handlers.go

@@ -4,15 +4,14 @@ import (
 	"bytes"
 	"fmt"
 	"io/ioutil"
-	"regexp"
 	"strconv"
 	"strings"
 	"time"
 
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/42wim/matterbridge/bridge/helper"
-	"github.com/dfordsoft/golib/ic"
 	"github.com/lrstanley/girc"
+	"github.com/missdeer/golib/ic"
 	"github.com/paulrosania/go-charset/charset"
 	"github.com/saintfish/chardet"
 
@@ -55,12 +54,12 @@ func (b *Birc) handleFiles(msg *config.Message) bool {
 	for _, f := range msg.Extra["file"] {
 		fi := f.(config.FileInfo)
 		if fi.Comment != "" {
-			msg.Text += fi.Comment + ": "
+			msg.Text += fi.Comment + " : "
 		}
 		if fi.URL != "" {
 			msg.Text = fi.URL
 			if fi.Comment != "" {
-				msg.Text = fi.Comment + ": " + fi.URL
+				msg.Text = fi.Comment + " : " + fi.URL
 			}
 		}
 		b.Local <- config.Message{Text: msg.Text, Username: msg.Username, Channel: msg.Channel, Event: msg.Event}
@@ -91,8 +90,13 @@ func (b *Birc) handleJoinPart(client *girc.Client, event girc.Event) {
 		if b.GetBool("nosendjoinpart") {
 			return
 		}
-		b.Log.Debugf("<= Sending JOIN_LEAVE event from %s to gateway", b.Account)
 		msg := config.Message{Username: "system", Text: event.Source.Name + " " + strings.ToLower(event.Command) + "s", Channel: channel, Account: b.Account, Event: config.EventJoinLeave}
+		if b.GetBool("verbosejoinpart") {
+			b.Log.Debugf("<= Sending verbose JOIN_LEAVE event from %s to gateway", b.Account)
+			msg = config.Message{Username: "system", Text: event.Source.Name + " (" + event.Source.Ident + "@" + event.Source.Host + ") " + strings.ToLower(event.Command) + "s", Channel: channel, Account: b.Account, Event: config.EventJoinLeave}
+		} else {
+			b.Log.Debugf("<= Sending JOIN_LEAVE event from %s to gateway", b.Account)
+		}
 		b.Log.Debugf("<= Message is %#v", msg)
 		b.Remote <- msg
 		return
@@ -156,7 +160,10 @@ func (b *Birc) handleOtherAuth(client *girc.Client, event girc.Event) {
 	b.handleNickServ()
 	b.handleRunCommands()
 	// we are now fully connected
-	b.connected <- nil
+	// only send on first connection
+	if b.FirstConnection {
+		b.connected <- nil
+	}
 }
 
 func (b *Birc) handlePrivMsg(client *girc.Client, event girc.Event) {
@@ -174,10 +181,6 @@ func (b *Birc) handlePrivMsg(client *girc.Client, event girc.Event) {
 	// strip action, we made an event if it was an action
 	rmsg.Text += event.StripAction()
 
-	// strip IRC colors
-	re := regexp.MustCompile(`\x03(?:\d{1,2}(?:,\d{1,2})?)?|[[:cntrl:]]`)
-	rmsg.Text = re.ReplaceAllString(rmsg.Text, "")
-
 	// start detecting the charset
 	mycharset := b.GetString("Charset")
 	if mycharset == "" {

bridge/irc/irc.go

@@ -14,6 +14,7 @@ import (
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/42wim/matterbridge/bridge/helper"
 	"github.com/lrstanley/girc"
+	stripmd "github.com/writeas/go-strip-markdown"
 
 	// We need to import the 'data' package as an implicit dependency.
 	// See: https://godoc.org/github.com/paulrosania/go-charset/charset
@@ -156,6 +157,10 @@ func (b *Birc) Send(msg config.Message) (string, error) {
 	}
 
 	var msgLines []string
+	if b.GetBool("StripMarkdown") {
+		msg.Text = stripmd.Strip(msg.Text)
+	}
+
 	if b.GetBool("MessageSplit") {
 		msgLines = helper.GetSubLines(msg.Text, b.MessageLength)
 	} else {
@@ -167,12 +172,8 @@ func (b *Birc) Send(msg config.Message) (string, error) {
 			return "", nil
 		}
 
-		b.Local <- config.Message{
-			Text:     msgLines[i],
-			Username: msg.Username,
-			Channel:  msg.Channel,
-			Event:    msg.Event,
-		}
+		msg.Text = msgLines[i]
+		b.Local <- msg
 	}
 	return "", nil
 }
@@ -205,7 +206,7 @@ func (b *Birc) doSend() {
 	for msg := range b.Local {
 		<-throttle.C
 		username := msg.Username
-		if b.GetBool("Colornicks") {
+		if b.GetBool("Colornicks") && len(username) > 1 {
 			checksum := crc32.ChecksumIEEE([]byte(msg.Username))
 			colorCode := checksum%14 + 2 // quick fix - prevent white or black color codes
 			username = fmt.Sprintf("\x03%02d%s\x0F", colorCode, msg.Username)
@@ -249,6 +250,8 @@ func (b *Birc) getClient() (*girc.Client, error) {
 		SSL:        b.GetBool("UseTLS"),
 		TLSConfig:  &tls.Config{InsecureSkipVerify: b.GetBool("SkipTLSVerify"), ServerName: server}, //nolint:gosec
 		PingDelay:  time.Minute,
+		// skip gIRC internal rate limiting, since we have our own throttling
+		AllowFlood: true,
 	})
 	return i, nil
 }

bridge/keybase/handlers.go

@@ -0,0 +1,59 @@
+package bkeybase
+
+import (
+	"strconv"
+
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/keybase/go-keybase-chat-bot/kbchat/types/chat1"
+)
+
+func (b *Bkeybase) handleKeybase() {
+	sub, err := b.kbc.ListenForNewTextMessages()
+	if err != nil {
+		b.Log.Errorf("Error listening: %s", err.Error())
+	}
+
+	go func() {
+		for {
+			msg, err := sub.Read()
+			if err != nil {
+				b.Log.Errorf("failed to read message: %s", err.Error())
+			}
+
+			if msg.Message.Content.TypeName != "text" {
+				continue
+			}
+
+			if msg.Message.Sender.Username == b.kbc.GetUsername() {
+				continue
+			}
+
+			b.handleMessage(msg.Message)
+
+		}
+	}()
+}
+
+func (b *Bkeybase) handleMessage(msg chat1.MsgSummary) {
+	b.Log.Debugf("== Receiving event: %#v", msg)
+	if msg.Channel.TopicName != b.channel || msg.Channel.Name != b.team {
+		return
+	}
+
+	if msg.Sender.Username != b.kbc.GetUsername() {
+
+		// TODO download avatar
+
+		// Create our message
+		rmsg := config.Message{Username: msg.Sender.Username, Text: msg.Content.Text.Body, UserID: string(msg.Sender.Uid), Channel: msg.Channel.TopicName, ID: strconv.Itoa(int(msg.Id)), Account: b.Account}
+
+		// Text must be a string
+		if msg.Content.TypeName != "text" {
+			b.Log.Errorf("message is not text")
+			return
+		}
+
+		b.Log.Debugf("<= Sending message from %s on %s to gateway", msg.Sender.Username, msg.Channel.Name)
+		b.Remote <- rmsg
+	}
+}

bridge/keybase/keybase.go

@@ -0,0 +1,106 @@
+package bkeybase
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+
+	"github.com/42wim/matterbridge/bridge"
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/keybase/go-keybase-chat-bot/kbchat"
+)
+
+// Bkeybase bridge structure
+type Bkeybase struct {
+	kbc     *kbchat.API
+	user    string
+	channel string
+	team    string
+	*bridge.Config
+}
+
+// New initializes Bkeybase object and sets team
+func New(cfg *bridge.Config) bridge.Bridger {
+	b := &Bkeybase{Config: cfg}
+	b.team = b.Config.GetString("Team")
+	return b
+}
+
+// Connect starts keybase API and listener loop
+func (b *Bkeybase) Connect() error {
+	var err error
+	b.Log.Infof("Connecting %s", b.GetString("Team"))
+
+	// use default keybase location (`keybase`)
+	b.kbc, err = kbchat.Start(kbchat.RunOptions{})
+	if err != nil {
+		return err
+	}
+	b.user = b.kbc.GetUsername()
+	b.Log.Info("Connection succeeded")
+	go b.handleKeybase()
+	return nil
+}
+
+// Disconnect doesn't do anything for now
+func (b *Bkeybase) Disconnect() error {
+	return nil
+}
+
+// JoinChannel sets channel name in struct
+func (b *Bkeybase) JoinChannel(channel config.ChannelInfo) error {
+	if _, err := b.kbc.JoinChannel(b.team, channel.Name); err != nil {
+		return err
+	}
+	b.channel = channel.Name
+	return nil
+}
+
+// Send receives bridge messages and sends them to Keybase chat room
+func (b *Bkeybase) Send(msg config.Message) (string, error) {
+	b.Log.Debugf("=> Receiving %#v", msg)
+
+	// Handle /me events
+	if msg.Event == config.EventUserAction {
+		msg.Text = "_" + msg.Text + "_"
+	}
+
+	// Delete message if we have an ID
+	// Delete message not supported by keybase go library yet
+
+	// Edit message if we have an ID
+	// kbchat lib does not support message editing yet
+
+	if len(msg.Extra["file"]) > 0 {
+		// Upload a file
+		dir, err := ioutil.TempDir("", "matterbridge")
+		if err != nil {
+			return "", err
+		}
+		defer os.RemoveAll(dir)
+
+		for _, f := range msg.Extra["file"] {
+			fname := f.(config.FileInfo).Name
+			fdata := *f.(config.FileInfo).Data
+			fcaption := f.(config.FileInfo).Comment
+			fpath := filepath.Join(dir, fname)
+
+			if err = ioutil.WriteFile(fpath, fdata, 0600); err != nil {
+				return "", err
+			}
+
+			_, _ = b.kbc.SendAttachmentByTeam(b.team, &b.channel, fpath, fcaption)
+		}
+
+		return "", nil
+	}
+
+	// Send regular message
+	text := msg.Username + msg.Text
+	resp, err := b.kbc.SendMessageByTeamName(b.team, &b.channel, text)
+	if err != nil {
+		return "", err
+	}
+	return strconv.Itoa(int(*resp.Result.MessageID)), err
+}

bridge/matrix/matrix.go

@@ -2,12 +2,14 @@ package bmatrix
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"html"
 	"mime"
 	"regexp"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/42wim/matterbridge/bridge"
 	"github.com/42wim/matterbridge/bridge/config"
@@ -20,13 +22,21 @@ type Bmatrix struct {
 	UserID  string
 	RoomMap map[string]string
 	sync.RWMutex
-	htmlTag *regexp.Regexp
+	htmlTag            *regexp.Regexp
+	htmlReplacementTag *regexp.Regexp
 	*bridge.Config
 }
 
+type httpError struct {
+	Errcode      string `json:"errcode"`
+	Err          string `json:"error"`
+	RetryAfterMs int    `json:"retry_after_ms"`
+}
+
 func New(cfg *bridge.Config) bridge.Bridger {
 	b := &Bmatrix{Config: cfg}
 	b.htmlTag = regexp.MustCompile("</.*?>")
+	b.htmlReplacementTag = regexp.MustCompile("<[^>]*>")
 	b.RoomMap = make(map[string]string)
 	return b
 }
@@ -58,14 +68,25 @@ func (b *Bmatrix) Disconnect() error {
 }
 
 func (b *Bmatrix) JoinChannel(channel config.ChannelInfo) error {
+retry:
 	resp, err := b.mc.JoinRoom(channel.Name, "", nil)
 	if err != nil {
+		httpErr := handleError(err)
+		if httpErr.Errcode == "M_LIMIT_EXCEEDED" {
+			b.Log.Infof("getting ratelimited by matrix, sleeping approx %d seconds before joining %s", httpErr.RetryAfterMs/1000, channel.Name)
+			time.Sleep((time.Duration(httpErr.RetryAfterMs) * time.Millisecond))
+
+			goto retry
+		}
+
 		return err
 	}
+
 	b.Lock()
 	b.RoomMap[resp.RoomID] = channel.Name
 	b.Unlock()
-	return err
+
+	return nil
 }
 
 func (b *Bmatrix) Send(msg config.Message) (string, error) {
@@ -124,13 +145,28 @@ func (b *Bmatrix) Send(msg config.Message) (string, error) {
 		return resp.EventID, err
 	}
 
-	username := html.EscapeString(msg.Username)
+	if b.GetBool("HTMLDisable") {
+		resp, err := b.mc.SendText(channel, msg.Username+msg.Text)
+		if err != nil {
+			return "", err
+		}
+		return resp.EventID, err
+	}
+
+	var username string
+	var plainUsername string
 	// check if we have a </tag>. if we have, we don't escape HTML. #696
 	if b.htmlTag.MatchString(msg.Username) {
 		username = msg.Username
+		// remove the HTML formatting for beautiful push messages #1188
+		plainUsername = b.htmlReplacementTag.ReplaceAllString(msg.Username, "")
+	} else {
+		username = html.EscapeString(msg.Username)
+		plainUsername = msg.Username
 	}
+
 	// Post normal message with HTML support (eg riot.im)
-	resp, err := b.mc.SendHTML(channel, msg.Username+msg.Text, username+helper.ParseMarkdown(msg.Text))
+	resp, err := b.mc.SendHTML(channel, plainUsername+msg.Text, username+helper.ParseMarkdown(msg.Text))
 	if err != nil {
 		return "", err
 	}
@@ -172,10 +208,15 @@ func (b *Bmatrix) handleEvent(ev *matrix.Event) {
 			return
 		}
 
-		// TODO download avatar
-
 		// Create our message
-		rmsg := config.Message{Username: ev.Sender[1:], Channel: channel, Account: b.Account, UserID: ev.Sender, ID: ev.ID}
+		rmsg := config.Message{
+			Username: ev.Sender[1:],
+			Channel:  channel,
+			Account:  b.Account,
+			UserID:   ev.Sender,
+			ID:       ev.ID,
+			Avatar:   b.getAvatarURL(ev.Sender),
+		}
 
 		// Text must be a string
 		if rmsg.Text, ok = ev.Content["body"].(string); !ok {
@@ -291,7 +332,8 @@ func (b *Bmatrix) handleUploadFile(msg *config.Message, channel string, fi *conf
 	content := bytes.NewReader(*fi.Data)
 	sp := strings.Split(fi.Name, ".")
 	mtype := mime.TypeByExtension("." + sp[len(sp)-1])
-	if !strings.Contains(mtype, "image") && !strings.Contains(mtype, "video") {
+	if !(strings.Contains(mtype, "image") || strings.Contains(mtype, "video") ||
+		strings.Contains(mtype, "application") || strings.Contains(mtype, "audio")) {
 		return
 	}
 	if fi.Comment != "" {
@@ -326,6 +368,18 @@ func (b *Bmatrix) handleUploadFile(msg *config.Message, channel string, fi *conf
 		if err != nil {
 			b.Log.Errorf("sendImage failed: %#v", err)
 		}
+	case strings.Contains(mtype, "application"):
+		b.Log.Debugf("sendFile %s", res.ContentURI)
+		_, err = b.mc.SendFile(channel, fi.Name, res.ContentURI, mtype, uint(len(*fi.Data)))
+		if err != nil {
+			b.Log.Errorf("sendFile failed: %#v", err)
+		}
+	case strings.Contains(mtype, "audio"):
+		b.Log.Debugf("sendAudio %s", res.ContentURI)
+		_, err = b.mc.SendAudio(channel, fi.Name, res.ContentURI, mtype, uint(len(*fi.Data)))
+		if err != nil {
+			b.Log.Errorf("sendAudio failed: %#v", err)
+		}
 	}
 	b.Log.Debugf("result: %#v", res)
 }
@@ -345,3 +399,36 @@ func (b *Bmatrix) containsAttachment(content map[string]interface{}) bool {
 	}
 	return true
 }
+
+// getAvatarURL returns the avatar URL of the specified sender
+func (b *Bmatrix) getAvatarURL(sender string) string {
+	mxcURL, err := b.mc.GetSenderAvatarURL(sender)
+	if err != nil {
+		b.Log.Errorf("getAvatarURL failed: %s", err)
+		return ""
+	}
+	url := strings.ReplaceAll(mxcURL, "mxc://", b.GetString("Server")+"/_matrix/media/r0/thumbnail/")
+	if url != "" {
+		url += "?width=37&height=37&method=crop"
+	}
+	return url
+}
+
+func handleError(err error) *httpError {
+	mErr, ok := err.(matrix.HTTPError)
+	if !ok {
+		return &httpError{
+			Err: "not a HTTPError",
+		}
+	}
+
+	var httpErr httpError
+
+	if err := json.Unmarshal(mErr.Contents, &httpErr); err != nil {
+		return &httpError{
+			Err: "unmarshal failed",
+		}
+	}
+
+	return &httpErr
+}

bridge/mattermost/handlers.go

@@ -4,7 +4,7 @@ import (
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/42wim/matterbridge/bridge/helper"
 	"github.com/42wim/matterbridge/matterclient"
-	"github.com/mattermost/mattermost-server/model"
+	"github.com/mattermost/mattermost-server/v5/model"
 )
 
 // handleDownloadAvatar downloads the avatar of userid from channel
@@ -66,6 +66,10 @@ func (b *Bmattermost) handleMatter() {
 		} else {
 			b.Log.Debugf("Choosing login/password based receiving")
 		}
+		// if for some reason we only want to sent stuff to mattermost but not receive, return
+		if b.GetString("WebhookBindAddress") == "" && b.GetString("WebhookURL") != "" && b.GetString("Token") == "" && b.GetString("Login") == "" {
+			b.Log.Debugf("No WebhookBindAddress specified, only WebhookURL. You will not receive messages from mattermost, only sending is possible.")
+		}
 		go b.handleMatterClient(messages)
 	}
 	var ok bool

bridge/mattermost/helpers.go

@@ -7,7 +7,7 @@ import (
 	"github.com/42wim/matterbridge/bridge/helper"
 	"github.com/42wim/matterbridge/matterclient"
 	"github.com/42wim/matterbridge/matterhook"
-	"github.com/mattermost/mattermost-server/model"
+	"github.com/mattermost/mattermost-server/v5/model"
 )
 
 func (b *Bmattermost) doConnectWebhookBind() error {
@@ -70,6 +70,7 @@ func (b *Bmattermost) apiLogin() error {
 		b.mc.SetLogLevel("debug")
 	}
 	b.mc.SkipTLSVerify = b.GetBool("SkipTLSVerify")
+	b.mc.SkipVersionCheck = b.GetBool("SkipVersionCheck")
 	b.mc.NoTLS = b.GetBool("NoTLS")
 	b.Log.Infof("Connecting %s (team: %s) on %s", b.GetString("Login"), b.GetString("Team"), b.GetString("Server"))
 	err := b.mc.Login()
@@ -186,6 +187,12 @@ func (b *Bmattermost) skipMessage(message *matterclient.Message) bool {
 		return true
 	}
 
+	// Ignore non-post messages
+	if message.Post == nil {
+		b.Log.Debugf("ignoring nil message.Post: %#v", message)
+		return true
+	}
+
 	// Ignore messages sent from matterbridge
 	if message.Post.Props != nil {
 		if _, ok := message.Post.Props["matterbridge_"+b.uuid].(bool); ok {

bridge/mattermost/mattermost.go

@@ -121,6 +121,12 @@ func (b *Bmattermost) Send(msg config.Message) (string, error) {
 		return msg.ID, b.mc.DeleteMessage(msg.ID)
 	}
 
+	// Handle prefix hint for unthreaded messages.
+	if msg.ParentID == "msg-parent-not-found" {
+		msg.ParentID = ""
+		msg.Text = fmt.Sprintf("[thread]: %s", msg.Text)
+	}
+
 	// Upload a file if it exists
 	if msg.Extra != nil {
 		for _, rmsg := range helper.HandleExtra(&msg, b.General) {

bridge/msteams/handler.go

@@ -0,0 +1,101 @@
+package bmsteams
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"strings"
+
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/42wim/matterbridge/bridge/helper"
+
+	msgraph "github.com/yaegashi/msgraph.go/beta"
+)
+
+func (b *Bmsteams) findFile(weburl string) (string, error) {
+	itemRB, err := b.gc.GetDriveItemByURL(b.ctx, weburl)
+	if err != nil {
+		return "", err
+	}
+	itemRB.Workbook().Worksheets()
+	b.gc.Workbooks()
+	item, err := itemRB.Request().Get(b.ctx)
+	if err != nil {
+		return "", err
+	}
+	if url, ok := item.GetAdditionalData("@microsoft.graph.downloadUrl"); ok {
+		return url.(string), nil
+	}
+	return "", nil
+}
+
+// handleDownloadFile handles file download
+func (b *Bmsteams) handleDownloadFile(rmsg *config.Message, filename, weburl string) error {
+	realURL, err := b.findFile(weburl)
+	if err != nil {
+		return err
+	}
+	// Actually download the file.
+	data, err := helper.DownloadFile(realURL)
+	if err != nil {
+		return fmt.Errorf("download %s failed %#v", weburl, err)
+	}
+
+	// If a comment is attached to the file(s) it is in the 'Text' field of the teams messge event
+	// and should be added as comment to only one of the files. We reset the 'Text' field to ensure
+	// that the comment is not duplicated.
+	comment := rmsg.Text
+	rmsg.Text = ""
+	helper.HandleDownloadData(b.Log, rmsg, filename, comment, weburl, data, b.General)
+	return nil
+}
+
+func (b *Bmsteams) handleAttachments(rmsg *config.Message, msg msgraph.ChatMessage) {
+	for _, a := range msg.Attachments {
+		//remove the attachment tags from the text
+		rmsg.Text = attachRE.ReplaceAllString(rmsg.Text, "")
+
+		//handle a code snippet (code block)
+		if *a.ContentType == "application/vnd.microsoft.card.codesnippet" {
+			b.handleCodeSnippet(rmsg, a)
+			continue
+		}
+
+		//handle the download
+		err := b.handleDownloadFile(rmsg, *a.Name, *a.ContentURL)
+		if err != nil {
+			b.Log.Errorf("download of %s failed: %s", *a.Name, err)
+		}
+	}
+}
+
+type AttachContent struct {
+	Language       string `json:"language"`
+	CodeSnippetURL string `json:"codeSnippetUrl"`
+}
+
+func (b *Bmsteams) handleCodeSnippet(rmsg *config.Message, attach msgraph.ChatMessageAttachment) {
+	var content AttachContent
+	err := json.Unmarshal([]byte(*attach.Content), &content)
+	if err != nil {
+		b.Log.Errorf("unmarshal codesnippet failed: %s", err)
+		return
+	}
+	s := strings.Split(content.CodeSnippetURL, "/")
+	if len(s) != 13 {
+		b.Log.Errorf("codesnippetUrl has unexpected size: %s", content.CodeSnippetURL)
+		return
+	}
+	resp, err := b.gc.Teams().Request().Client().Get(content.CodeSnippetURL)
+	if err != nil {
+		b.Log.Errorf("retrieving snippet content failed:%s", err)
+		return
+	}
+	defer resp.Body.Close()
+	res, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		b.Log.Errorf("reading snippet data failed: %s", err)
+		return
+	}
+	rmsg.Text = rmsg.Text + "\n```" + content.Language + "\n" + string(res) + "\n```\n"
+}

bridge/msteams/msteams.go

@@ -0,0 +1,224 @@
+package bmsteams
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/42wim/matterbridge/bridge"
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/davecgh/go-spew/spew"
+
+	"github.com/mattn/godown"
+	msgraph "github.com/yaegashi/msgraph.go/beta"
+	"github.com/yaegashi/msgraph.go/msauth"
+
+	"golang.org/x/oauth2"
+)
+
+var defaultScopes = []string{"openid", "profile", "offline_access", "Group.Read.All", "Group.ReadWrite.All"}
+var attachRE = regexp.MustCompile(`<attachment id=.*?attachment>`)
+
+type Bmsteams struct {
+	gc    *msgraph.GraphServiceRequestBuilder
+	ctx   context.Context
+	botID string
+	*bridge.Config
+}
+
+func New(cfg *bridge.Config) bridge.Bridger {
+	return &Bmsteams{Config: cfg}
+}
+
+func (b *Bmsteams) Connect() error {
+	tokenCachePath := b.GetString("sessionFile")
+	if tokenCachePath == "" {
+		tokenCachePath = "msteams_session.json"
+	}
+	ctx := context.Background()
+	m := msauth.NewManager()
+	m.LoadFile(tokenCachePath) //nolint:errcheck
+	ts, err := m.DeviceAuthorizationGrant(ctx, b.GetString("TenantID"), b.GetString("ClientID"), defaultScopes, nil)
+	if err != nil {
+		return err
+	}
+	err = m.SaveFile(tokenCachePath)
+	if err != nil {
+		b.Log.Errorf("Couldn't save sessionfile in %s: %s", tokenCachePath, err)
+	}
+	// make file readable only for matterbridge user
+	err = os.Chmod(tokenCachePath, 0600)
+	if err != nil {
+		b.Log.Errorf("Couldn't change permissions for %s: %s", tokenCachePath, err)
+	}
+	httpClient := oauth2.NewClient(ctx, ts)
+	graphClient := msgraph.NewClient(httpClient)
+	b.gc = graphClient
+	b.ctx = ctx
+
+	err = b.setBotID()
+	if err != nil {
+		return err
+	}
+	b.Log.Info("Connection succeeded")
+	return nil
+}
+
+func (b *Bmsteams) Disconnect() error {
+	return nil
+}
+
+func (b *Bmsteams) JoinChannel(channel config.ChannelInfo) error {
+	go func(name string) {
+		for {
+			err := b.poll(name)
+			if err != nil {
+				b.Log.Errorf("polling failed for %s: %s. retrying in 5 seconds", name, err)
+			}
+			time.Sleep(time.Second * 5)
+		}
+	}(channel.Name)
+	return nil
+}
+
+func (b *Bmsteams) Send(msg config.Message) (string, error) {
+	b.Log.Debugf("=> Receiving %#v", msg)
+	if msg.ParentID != "" && msg.ParentID != "msg-parent-not-found" {
+		return b.sendReply(msg)
+	}
+	if msg.ParentID == "msg-parent-not-found" {
+		msg.ParentID = ""
+		msg.Text = fmt.Sprintf("[thread]: %s", msg.Text)
+	}
+	ct := b.gc.Teams().ID(b.GetString("TeamID")).Channels().ID(msg.Channel).Messages().Request()
+	text := msg.Username + msg.Text
+	content := &msgraph.ItemBody{Content: &text}
+	rmsg := &msgraph.ChatMessage{Body: content}
+	res, err := ct.Add(b.ctx, rmsg)
+	if err != nil {
+		return "", err
+	}
+	return *res.ID, nil
+}
+
+func (b *Bmsteams) sendReply(msg config.Message) (string, error) {
+	ct := b.gc.Teams().ID(b.GetString("TeamID")).Channels().ID(msg.Channel).Messages().ID(msg.ParentID).Replies().Request()
+	// Handle prefix hint for unthreaded messages.
+
+	text := msg.Username + msg.Text
+	content := &msgraph.ItemBody{Content: &text}
+	rmsg := &msgraph.ChatMessage{Body: content}
+	res, err := ct.Add(b.ctx, rmsg)
+	if err != nil {
+		return "", err
+	}
+	return *res.ID, nil
+}
+
+func (b *Bmsteams) getMessages(channel string) ([]msgraph.ChatMessage, error) {
+	ct := b.gc.Teams().ID(b.GetString("TeamID")).Channels().ID(channel).Messages().Request()
+	rct, err := ct.Get(b.ctx)
+	if err != nil {
+		return nil, err
+	}
+	b.Log.Debugf("got %#v messages", len(rct))
+	return rct, nil
+}
+
+//nolint:gocognit
+func (b *Bmsteams) poll(channelName string) error {
+	msgmap := make(map[string]time.Time)
+	b.Log.Debug("getting initial messages")
+	res, err := b.getMessages(channelName)
+	if err != nil {
+		return err
+	}
+	for _, msg := range res {
+		msgmap[*msg.ID] = *msg.CreatedDateTime
+		if msg.LastModifiedDateTime != nil {
+			msgmap[*msg.ID] = *msg.LastModifiedDateTime
+		}
+	}
+	time.Sleep(time.Second * 5)
+	b.Log.Debug("polling for messages")
+	for {
+		res, err := b.getMessages(channelName)
+		if err != nil {
+			return err
+		}
+		for i := len(res) - 1; i >= 0; i-- {
+			msg := res[i]
+			if mtime, ok := msgmap[*msg.ID]; ok {
+				if mtime == *msg.CreatedDateTime && msg.LastModifiedDateTime == nil {
+					continue
+				}
+				if msg.LastModifiedDateTime != nil && mtime == *msg.LastModifiedDateTime {
+					continue
+				}
+			}
+
+			if b.GetBool("debug") {
+				b.Log.Debug("Msg dump: ", spew.Sdump(msg))
+			}
+
+			// skip non-user message for now.
+			if msg.From.User == nil {
+				continue
+			}
+
+			if *msg.From.User.ID == b.botID {
+				b.Log.Debug("skipping own message")
+				msgmap[*msg.ID] = *msg.CreatedDateTime
+				continue
+			}
+
+			msgmap[*msg.ID] = *msg.CreatedDateTime
+			if msg.LastModifiedDateTime != nil {
+				msgmap[*msg.ID] = *msg.LastModifiedDateTime
+			}
+			b.Log.Debugf("<= Sending message from %s on %s to gateway", *msg.From.User.DisplayName, b.Account)
+			text := b.convertToMD(*msg.Body.Content)
+			rmsg := config.Message{
+				Username: *msg.From.User.DisplayName,
+				Text:     text,
+				Channel:  channelName,
+				Account:  b.Account,
+				Avatar:   "",
+				UserID:   *msg.From.User.ID,
+				ID:       *msg.ID,
+				Extra:    make(map[string][]interface{}),
+			}
+
+			b.handleAttachments(&rmsg, msg)
+			b.Log.Debugf("<= Message is %#v", rmsg)
+			b.Remote <- rmsg
+		}
+		time.Sleep(time.Second * 5)
+	}
+}
+
+func (b *Bmsteams) setBotID() error {
+	req := b.gc.Me().Request()
+	r, err := req.Get(b.ctx)
+	if err != nil {
+		return err
+	}
+	b.botID = *r.ID
+	return nil
+}
+
+func (b *Bmsteams) convertToMD(text string) string {
+	if !strings.Contains(text, "<div>") {
+		return text
+	}
+	var sb strings.Builder
+	err := godown.Convert(&sb, strings.NewReader(text), nil)
+	if err != nil {
+		b.Log.Errorf("Couldn't convert message to markdown %s", text)
+		return text
+	}
+	return sb.String()
+}

bridge/nctalk/nctalk.go

@@ -0,0 +1,114 @@
+package nctalk
+
+import (
+	"context"
+	"strconv"
+
+	"github.com/42wim/matterbridge/bridge"
+	"github.com/42wim/matterbridge/bridge/config"
+
+	talk "gomod.garykim.dev/nc-talk"
+	"gomod.garykim.dev/nc-talk/ocs"
+	"gomod.garykim.dev/nc-talk/room"
+	"gomod.garykim.dev/nc-talk/user"
+)
+
+type Btalk struct {
+	user  *user.TalkUser
+	rooms []Broom
+	*bridge.Config
+}
+
+func New(cfg *bridge.Config) bridge.Bridger {
+	return &Btalk{Config: cfg}
+}
+
+type Broom struct {
+	room      *room.TalkRoom
+	ctx       context.Context
+	ctxCancel context.CancelFunc
+}
+
+func (b *Btalk) Connect() error {
+	b.Log.Info("Connecting")
+	b.user = talk.NewUser(b.GetString("Server"), b.GetString("Login"), b.GetString("Password"))
+	_, err := b.user.Capabilities()
+	if err != nil {
+		b.Log.Error("Cannot Connect")
+		return err
+	}
+	b.Log.Info("Connected")
+	return nil
+}
+
+func (b *Btalk) Disconnect() error {
+	for _, r := range b.rooms {
+		r.ctxCancel()
+	}
+	return nil
+}
+
+func (b *Btalk) JoinChannel(channel config.ChannelInfo) error {
+	newRoom := Broom{
+		room: talk.NewRoom(b.user, channel.Name),
+	}
+	newRoom.ctx, newRoom.ctxCancel = context.WithCancel(context.Background())
+	c, err := newRoom.room.ReceiveMessages(newRoom.ctx)
+	if err != nil {
+		return err
+	}
+	b.rooms = append(b.rooms, newRoom)
+	go func() {
+		for msg := range c {
+			// ignore messages that are one of the following
+			// * not a message from a user
+			// * from ourselves
+			if msg.MessageType != ocs.MessageComment || msg.ActorID == b.user.User {
+				continue
+			}
+			remoteMessage := config.Message{
+				Text:     msg.Message,
+				Channel:  newRoom.room.Token,
+				Username: msg.ActorDisplayName,
+				UserID:   msg.ActorID,
+				Account:  b.Account,
+			}
+			// It is possible for the ID to not be set on older versions of Talk so we only set it if
+			// the ID is not blank
+			if msg.ID != 0 {
+				remoteMessage.ID = strconv.Itoa(msg.ID)
+			}
+			b.Log.Debugf("<= Message is %#v", remoteMessage)
+			b.Remote <- remoteMessage
+		}
+	}()
+	return nil
+}
+
+func (b *Btalk) Send(msg config.Message) (string, error) {
+	r := b.getRoom(msg.Channel)
+	if r == nil {
+		b.Log.Errorf("Could not find room for %v", msg.Channel)
+		return "", nil
+	}
+
+	// Talk currently only supports sending normal messages
+	if msg.Event != "" {
+		return "", nil
+	}
+	sentMessage, err := r.room.SendMessage(msg.Username + msg.Text)
+	if err != nil {
+		b.Log.Errorf("Could not send message to room %v from %v: %v", msg.Channel, msg.Username, err)
+		return "", nil
+	}
+	return strconv.Itoa(sentMessage.ID), nil
+}
+
+func (b *Btalk) getRoom(token string) *Broom {
+	for _, r := range b.rooms {
+		if r.room.Token == token {
+			return &r
+		}
+	}
+	return nil
+}

bridge/rocketchat/handlers.go

@@ -2,6 +2,7 @@ package brocketchat
 
 import (
 	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/matterbridge/Rocket.Chat.Go.SDK/models"
 )
 
 func (b *Brocketchat) handleRocket() {
@@ -38,6 +39,23 @@ func (b *Brocketchat) handleRocketHook(messages chan *config.Message) {
 	}
 }
 
+func (b *Brocketchat) handleStatusEvent(ev models.Message, rmsg *config.Message) bool {
+	switch ev.Type {
+	case "":
+		// this is a normal message, no processing needed
+		// return true so the message is not dropped
+		return true
+	case sUserJoined, sUserLeft:
+		rmsg.Event = config.EventJoinLeave
+		return true
+	case sRoomChangedTopic:
+		rmsg.Event = config.EventTopicChange
+		return true
+	}
+	b.Log.Debugf("Dropping message with unknown type: %s", ev.Type)
+	return false
+}
+
 func (b *Brocketchat) handleRocketClient(messages chan *config.Message) {
 	for message := range b.messageChan {
 		// skip messages with same ID, apparently messages get duplicated for an unknown reason
@@ -59,7 +77,12 @@ func (b *Brocketchat) handleRocketClient(messages chan *config.Message) {
 			UserID:   message.User.ID,
 			ID:       message.ID,
 		}
-		messages <- rmsg
+
+		// handleStatusEvent returns false if the message should be dropped
+		// in that case it is probably some modification to the channel we do not want to relay
+		if b.handleStatusEvent(m, rmsg) {
+			messages <- rmsg
+		}
 	}
 }
 

bridge/rocketchat/helpers.go

@@ -58,6 +58,9 @@ func (b *Brocketchat) doConnectWebhookURL() error {
 func (b *Brocketchat) apiLogin() error {
 	b.Log.Debugf("handling apiLogin()")
 	credentials := &models.UserCredentials{Email: b.GetString("login"), Password: b.GetString("password")}
+	if b.GetString("Token") != "" {
+		credentials = &models.UserCredentials{ID: b.GetString("Login"), Token: b.GetString("Token")}
+	}
 	myURL, err := url.Parse(b.GetString("server"))
 	if err != nil {
 		return err

bridge/rocketchat/rocketchat.go

@@ -29,6 +29,12 @@ type Brocketchat struct {
 	sync.RWMutex
 }
 
+const (
+	sUserJoined       = "uj"
+	sUserLeft         = "ul"
+	sRoomChangedTopic = "room_changed_topic"
+)
+
 func New(cfg *bridge.Config) bridge.Bridger {
 	newCache, err := lru.New(100)
 	if err != nil {
@@ -108,6 +114,11 @@ func (b *Brocketchat) Send(msg config.Message) (string, error) {
 	msg.Channel = strings.TrimPrefix(msg.Channel, "#")
 	channel := &models.Channel{ID: b.getChannelID(msg.Channel), Name: msg.Channel}
 
+	// Make a action /me of the message
+	if msg.Event == config.EventUserAction {
+		msg.Text = "_" + msg.Text + "_"
+	}
+
 	// Delete message
 	if msg.Event == config.EventMsgDelete {
 		if msg.ID == "" {

bridge/slack/handlers.go

@@ -1,18 +1,22 @@
 package bslack
 
 import (
+	"errors"
 	"fmt"
 	"html"
 	"time"
 
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/42wim/matterbridge/bridge/helper"
-	"github.com/nlopes/slack"
+	"github.com/slack-go/slack"
 )
 
+// ErrEventIgnored is for events that should be ignored
+var ErrEventIgnored = errors.New("this event message should ignored")
+
 func (b *Bslack) handleSlack() {
 	messages := make(chan *config.Message)
-	if b.GetString(incomingWebhookConfig) != "" {
+	if b.GetString(incomingWebhookConfig) != "" && b.GetString(tokenConfig) == "" {
 		b.Log.Debugf("Choosing webhooks based receiving")
 		go b.handleMatterHook(messages)
 	} else {
@@ -22,20 +26,21 @@ func (b *Bslack) handleSlack() {
 	time.Sleep(time.Second)
 	b.Log.Debug("Start listening for Slack messages")
 	for message := range messages {
-		if message.Event != config.EventUserTyping {
+		// don't do any action on deleted/typing messages
+		if message.Event != config.EventUserTyping && message.Event != config.EventMsgDelete {
 			b.Log.Debugf("<= Sending message from %s on %s to gateway", message.Username, b.Account)
+			// cleanup the message
+			message.Text = b.replaceMention(message.Text)
+			message.Text = b.replaceVariable(message.Text)
+			message.Text = b.replaceChannel(message.Text)
+			message.Text = b.replaceURL(message.Text)
+			message.Text = b.replaceb0rkedMarkDown(message.Text)
+			message.Text = html.UnescapeString(message.Text)
+
+			// Add the avatar
+			message.Avatar = b.users.getAvatar(message.UserID)
 		}
 
-		// cleanup the message
-		message.Text = b.replaceMention(message.Text)
-		message.Text = b.replaceVariable(message.Text)
-		message.Text = b.replaceChannel(message.Text)
-		message.Text = b.replaceURL(message.Text)
-		message.Text = html.UnescapeString(message.Text)
-
-		// Add the avatar
-		message.Avatar = b.users.getAvatar(message.UserID)
-
 		b.Log.Debugf("<= Message is %#v", message)
 		b.Remote <- *message
 	}
@@ -43,7 +48,7 @@ func (b *Bslack) handleSlack() {
 
 func (b *Bslack) handleSlackClient(messages chan *config.Message) {
 	for msg := range b.rtm.IncomingEvents {
-		if msg.Type != sUserTyping && msg.Type != sLatencyReport {
+		if msg.Type != sUserTyping && msg.Type != sHello && msg.Type != sLatencyReport {
 			b.Log.Debugf("== Receiving event %#v", msg.Data)
 		}
 		switch ev := msg.Data.(type) {
@@ -52,7 +57,9 @@ func (b *Bslack) handleSlackClient(messages chan *config.Message) {
 				continue
 			}
 			rmsg, err := b.handleTypingEvent(ev)
-			if err != nil {
+			if err == ErrEventIgnored {
+				continue
+			} else if err != nil {
 				b.Log.Errorf("%#v", err)
 				continue
 			}
@@ -86,7 +93,7 @@ func (b *Bslack) handleSlackClient(messages chan *config.Message) {
 			b.Log.Errorf("Connection failed %#v %#v", ev.Error(), ev.ErrorObj)
 		case *slack.MemberJoinedChannelEvent:
 			b.users.populateUser(ev.User)
-		case *slack.LatencyReport:
+		case *slack.HelloEvent, *slack.LatencyReport, *slack.ConnectingEvent:
 			continue
 		default:
 			b.Log.Debugf("Unhandled incoming event: %T", ev)
@@ -123,11 +130,11 @@ func (b *Bslack) skipMessageEvent(ev *slack.MessageEvent) bool {
 		}
 	}
 
-	// Skip any messages that we made ourselves or from 'slackbot' (see #527).
-	if ev.Username == sSlackBotUser ||
-		(b.rtm != nil && ev.Username == b.si.User.Name) ||
-		(len(ev.Attachments) > 0 && ev.Attachments[0].CallbackID == "matterbridge_"+b.uuid) {
-		return true
+	// Check for our callback ID
+	hasOurCallbackID := false
+	if len(ev.Blocks.BlockSet) == 1 {
+		block, ok := ev.Blocks.BlockSet[0].(*slack.SectionBlock)
+		hasOurCallbackID = ok && block.BlockID == "matterbridge_"+b.uuid
 	}
 
 	if ev.SubMessage != nil {
@@ -142,6 +149,16 @@ func (b *Bslack) skipMessageEvent(ev *slack.MessageEvent) bool {
 		if ev.SubType == "message_replied" && ev.Hidden {
 			return true
 		}
+		if len(ev.SubMessage.Blocks.BlockSet) == 1 {
+			block, ok := ev.SubMessage.Blocks.BlockSet[0].(*slack.SectionBlock)
+			hasOurCallbackID = ok && block.BlockID == "matterbridge_"+b.uuid
+		}
+	}
+
+	// Skip any messages that we made ourselves or from 'slackbot' (see #527).
+	if ev.Username == sSlackBotUser ||
+		(b.rtm != nil && ev.Username == b.si.User.Name) || hasOurCallbackID {
+		return true
 	}
 
 	if len(ev.Files) > 0 {
@@ -269,6 +286,9 @@ func (b *Bslack) handleAttachments(ev *slack.MessageEvent, rmsg *config.Message)
 }
 
 func (b *Bslack) handleTypingEvent(ev *slack.UserTypingEvent) (*config.Message, error) {
+	if ev.User == b.si.User.ID {
+		return nil, ErrEventIgnored
+	}
 	channelInfo, err := b.channels.getChannelByID(ev.Channel)
 	if err != nil {
 		return nil, err

bridge/slack/helpers.go

@@ -7,8 +7,8 @@ import (
 	"time"
 
 	"github.com/42wim/matterbridge/bridge/config"
-	"github.com/nlopes/slack"
 	"github.com/sirupsen/logrus"
+	"github.com/slack-go/slack"
 )
 
 // populateReceivedMessage shapes the initial Matterbridge message that we will forward to the
@@ -188,6 +188,36 @@ func (b *Bslack) replaceURL(text string) string {
 	return text
 }
 
+func (b *Bslack) replaceb0rkedMarkDown(text string) string {
+	// taken from https://github.com/mattermost/mattermost-server/blob/master/app/slackimport.go
+	//
+	regexReplaceAllString := []struct {
+		regex *regexp.Regexp
+		rpl   string
+	}{
+		// bold
+		{
+			regexp.MustCompile(`(^|[\s.;,])\*(\S[^*\n]+)\*`),
+			"$1**$2**",
+		},
+		// strikethrough
+		{
+			regexp.MustCompile(`(^|[\s.;,])\~(\S[^~\n]+)\~`),
+			"$1~~$2~~",
+		},
+		// single paragraph blockquote
+		// Slack converts > character to >
+		{
+			regexp.MustCompile(`(?sm)^>`),
+			">",
+		},
+	}
+	for _, rule := range regexReplaceAllString {
+		text = rule.regex.ReplaceAllString(text, rule.rpl)
+	}
+	return text
+}
+
 func (b *Bslack) replaceCodeFence(text string) string {
 	return codeFenceRE.ReplaceAllString(text, "```")
 }

bridge/slack/legacy.go

@@ -5,7 +5,7 @@ import (
 
 	"github.com/42wim/matterbridge/bridge"
 	"github.com/42wim/matterbridge/matterhook"
-	"github.com/nlopes/slack"
+	"github.com/slack-go/slack"
 )
 
 type BLegacy struct {
@@ -13,7 +13,9 @@ type BLegacy struct {
 }
 
 func NewLegacy(cfg *bridge.Config) bridge.Bridger {
-	return &BLegacy{Bslack: newBridge(cfg)}
+	b := &BLegacy{Bslack: newBridge(cfg)}
+	b.legacy = true
+	return b
 }
 
 func (b *BLegacy) Connect() error {

bridge/slack/slack.go

@@ -12,9 +12,9 @@ import (
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/42wim/matterbridge/bridge/helper"
 	"github.com/42wim/matterbridge/matterhook"
-	"github.com/hashicorp/golang-lru"
-	"github.com/nlopes/slack"
+	lru "github.com/hashicorp/golang-lru"
 	"github.com/rs/xid"
+	"github.com/slack-go/slack"
 )
 
 type Bslack struct {
@@ -32,9 +32,11 @@ type Bslack struct {
 
 	channels *channels
 	users    *users
+	legacy   bool
 }
 
 const (
+	sHello           = "hello"
 	sChannelJoin     = "channel_join"
 	sChannelLeave    = "channel_leave"
 	sChannelJoined   = "channel_joined"
@@ -62,6 +64,7 @@ const (
 	editSuffixConfig      = "EditSuffix"
 	iconURLConfig         = "iconurl"
 	noSendJoinConfig      = "nosendjoinpart"
+	messageLength         = 3000
 )
 
 func New(cfg *bridge.Config) bridge.Bridger {
@@ -151,6 +154,18 @@ func (b *Bslack) JoinChannel(channel config.ChannelInfo) error {
 		return nil
 	}
 
+	// try to join a channel when in legacy
+	if b.legacy {
+		_, err := b.sc.JoinChannel(channel.Name)
+		if err != nil {
+			switch err.Error() {
+			case "name_taken", "restricted_action":
+			case "default":
+				return err
+			}
+		}
+	}
+
 	b.channels.populateChannels(false)
 
 	channelInfo, err := b.channels.getChannel(channel.Name)
@@ -163,7 +178,8 @@ func (b *Bslack) JoinChannel(channel config.ChannelInfo) error {
 		channel.Name = channelInfo.Name
 	}
 
-	if !channelInfo.IsMember {
+	// we can't join a channel unless we are using legacy tokens #651
+	if !channelInfo.IsMember && !b.legacy {
 		return fmt.Errorf("slack integration that matterbridge is using is not member of channel '%s', please add it manually", channelInfo.Name)
 	}
 	return nil
@@ -179,6 +195,7 @@ func (b *Bslack) Send(msg config.Message) (string, error) {
 		b.Log.Debugf("=> Receiving %#v", msg)
 	}
 
+	msg.Text = helper.ClipMessage(msg.Text, messageLength)
 	msg.Text = b.replaceCodeFence(msg.Text)
 
 	// Make a action /me of the message
@@ -187,7 +204,7 @@ func (b *Bslack) Send(msg config.Message) (string, error) {
 	}
 
 	// Use webhook to send the message
-	if b.GetString(outgoingWebhookConfig) != "" {
+	if b.GetString(outgoingWebhookConfig) != "" && b.GetString(tokenConfig) == "" {
 		return "", b.sendWebhook(msg)
 	}
 	return b.sendRTM(msg)
@@ -393,7 +410,6 @@ func (b *Bslack) editMessage(msg *config.Message, channelInfo *slack.Channel) (b
 	}
 	messageOptions := b.prepareMessageOptions(msg)
 	for {
-		messageOptions = append(messageOptions, slack.MsgOptionText(msg.Text, false))
 		_, _, _, err := b.rtm.UpdateMessage(channelInfo.ID, msg.ID, messageOptions...)
 		if err == nil {
 			return true, nil
@@ -412,11 +428,6 @@ func (b *Bslack) postMessage(msg *config.Message, channelInfo *slack.Channel) (s
 		return "", nil
 	}
 	messageOptions := b.prepareMessageOptions(msg)
-	messageOptions = append(
-		messageOptions,
-		slack.MsgOptionText(msg.Text, false),
-		slack.MsgOptionEnableLinkUnfurl(),
-	)
 	for {
 		_, id, err := b.rtm.PostMessage(channelInfo.ID, messageOptions...)
 		if err == nil {
@@ -482,8 +493,6 @@ func (b *Bslack) prepareMessageOptions(msg *config.Message) []slack.MsgOption {
 	}
 
 	var attachments []slack.Attachment
-	// add a callback ID so we can see we created it
-	attachments = append(attachments, slack.Attachment{CallbackID: "matterbridge_" + b.uuid})
 	// add file attachments
 	attachments = append(attachments, b.createAttach(msg.Extra)...)
 	// add slack attachments (from another slack bridge)
@@ -494,6 +503,19 @@ func (b *Bslack) prepareMessageOptions(msg *config.Message) []slack.MsgOption {
 	}
 
 	var opts []slack.MsgOption
+	opts = append(opts,
+		// provide regular text field (fallback used in Slack notifications, etc.)
+		slack.MsgOptionText(msg.Text, false),
+
+		// add a callback ID so we can see we created it
+		slack.MsgOptionBlocks(slack.NewSectionBlock(
+			slack.NewTextBlockObject(slack.MarkdownType, msg.Text, false, false),
+			nil, nil,
+			slack.SectionBlockOptionBlockID("matterbridge_"+b.uuid),
+		)),
+
+		slack.MsgOptionEnableLinkUnfurl(),
+	)
 	opts = append(opts, slack.MsgOptionAttachments(attachments...))
 	opts = append(opts, slack.MsgOptionPostMessageParameters(params))
 	return opts

bridge/slack/users_channels.go

@@ -8,8 +8,8 @@ import (
 	"time"
 
 	"github.com/42wim/matterbridge/bridge/config"
-	"github.com/nlopes/slack"
 	"github.com/sirupsen/logrus"
+	"github.com/slack-go/slack"
 )
 
 const minimumRefreshInterval = 10 * time.Second
@@ -87,6 +87,11 @@ func (b *users) populateUser(userID string) {
 			// in case the previous query failed for some reason.
 		} else {
 			b.usersSyncPoints[userID] = make(chan struct{})
+			defer func() {
+				// Wake up any waiting goroutines and remove the synchronization point.
+				close(b.usersSyncPoints[userID])
+				delete(b.usersSyncPoints, userID)
+			}()
 			break
 		}
 	}
@@ -106,10 +111,6 @@ func (b *users) populateUser(userID string) {
 
 	// Register user information.
 	b.users[userID] = user
-
-	// Wake up any waiting goroutines and remove the synchronization point.
-	close(b.usersSyncPoints[userID])
-	delete(b.usersSyncPoints, userID)
 }
 
 func (b *users) populateUsers(wait bool) {

bridge/sshchat/sshchat.go

@@ -130,6 +130,10 @@ func (b *Bsshchat) handleSSHChat() error {
 			if strings.Contains(b.r.Text(), "Rate limiting is in effect") {
 				continue
 			}
+			// skip our own messages
+			if !strings.HasPrefix(b.r.Text(), "["+b.GetString("Nick")+"] \x1b") {
+				continue
+			}
 			res := strings.Split(stripPrompt(b.r.Text()), ":")
 			if res[0] == "-> Set theme" {
 				wait = false

bridge/steam/handlers.go

@@ -85,7 +85,7 @@ func (b *Bsteam) handleEvents() {
 
 func (b *Bsteam) handleLogOnFailed(e *steam.LogOnFailedEvent, myLoginInfo *steam.LogOnDetails) error {
 	switch e.Result {
-	case steamlang.EResult_AccountLogonDeniedNeedTwoFactorCode:
+	case steamlang.EResult_AccountLoginDeniedNeedTwoFactor:
 		b.Log.Info("Steam guard isn't letting me in! Enter 2FA code:")
 		var code string
 		fmt.Scanf("%s", &code)

bridge/telegram/handlers.go

@@ -5,10 +5,11 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
+	"unicode/utf16"
 
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/42wim/matterbridge/bridge/helper"
-	"github.com/go-telegram-bot-api/telegram-bot-api"
+	tgbotapi "github.com/go-telegram-bot-api/telegram-bot-api"
 )
 
 func (b *Btelegram) handleUpdate(rmsg *config.Message, message, posted, edited *tgbotapi.Message) *tgbotapi.Message {
@@ -38,22 +39,32 @@ func (b *Btelegram) handleGroups(rmsg *config.Message, message *tgbotapi.Message
 
 // handleForwarded handles forwarded messages
 func (b *Btelegram) handleForwarded(rmsg *config.Message, message *tgbotapi.Message) {
-	if message.ForwardFrom != nil {
-		usernameForward := ""
-		if b.GetBool("UseFirstName") {
+	if message.ForwardDate == 0 {
+		return
+	}
+
+	if message.ForwardFrom == nil {
+		rmsg.Text = "Forwarded from " + unknownUser + ": " + rmsg.Text
+		return
+	}
+
+	usernameForward := ""
+	if b.GetBool("UseFirstName") {
+		usernameForward = message.ForwardFrom.FirstName
+	}
+
+	if usernameForward == "" {
+		usernameForward = message.ForwardFrom.UserName
+		if usernameForward == "" {
 			usernameForward = message.ForwardFrom.FirstName
 		}
-		if usernameForward == "" {
-			usernameForward = message.ForwardFrom.UserName
-			if usernameForward == "" {
-				usernameForward = message.ForwardFrom.FirstName
-			}
-		}
-		if usernameForward == "" {
-			usernameForward = unknownUser
-		}
-		rmsg.Text = "Forwarded from " + usernameForward + ": " + rmsg.Text
 	}
+
+	if usernameForward == "" {
+		usernameForward = unknownUser
+	}
+
+	rmsg.Text = "Forwarded from " + usernameForward + ": " + rmsg.Text
 }
 
 // handleQuoting handles quoting of previous messages
@@ -94,7 +105,7 @@ func (b *Btelegram) handleUsername(rmsg *config.Message, message *tgbotapi.Messa
 			}
 		}
 		// only download avatars if we have a place to upload them (configured mediaserver)
-		if b.General.MediaServerUpload != "" {
+		if b.General.MediaServerUpload != "" || (b.General.MediaServerDownload != "" && b.General.MediaDownloadPath != "") {
 			b.handleDownloadAvatar(message.From.ID, rmsg.Channel)
 		}
 	}
@@ -206,6 +217,46 @@ func (b *Btelegram) handleDownloadAvatar(userid int, channel string) {
 	}
 }
 
+func (b *Btelegram) maybeConvertTgs(name *string, data *[]byte) {
+	var format string
+	switch b.GetString("MediaConvertTgs") {
+	case FormatWebp:
+		b.Log.Debugf("Tgs to WebP conversion enabled, converting %v", name)
+		format = FormatWebp
+	case FormatPng:
+		// The WebP to PNG converter can't handle animated webp files yet,
+		// and I'm not going to write a path for x/image/webp.
+		// The error message would be:
+		//     conversion failed: webp: non-Alpha VP8X is not implemented
+		// So instead, we tell lottie to directly go to PNG.
+		b.Log.Debugf("Tgs to PNG conversion enabled, converting %v", name)
+		format = FormatPng
+	default:
+		// Otherwise, no conversion was requested. Trying to run the usual webp
+		// converter would fail, because '.tgs.webp' is actually a gzipped JSON
+		// file, and has nothing to do with WebP.
+		return
+	}
+	err := helper.ConvertTgsToX(data, format, b.Log)
+	if err != nil {
+		b.Log.Errorf("conversion failed: %v", err)
+	} else {
+		*name = strings.Replace(*name, "tgs.webp", format, 1)
+	}
+}
+
+func (b *Btelegram) maybeConvertWebp(name *string, data *[]byte) {
+	if b.GetBool("MediaConvertWebPToPNG") {
+		b.Log.Debugf("WebP to PNG conversion enabled, converting %v", name)
+		err := helper.ConvertWebPToPNG(data)
+		if err != nil {
+			b.Log.Errorf("conversion failed: %v", err)
+		} else {
+			*name = strings.Replace(*name, ".webp", ".png", 1)
+		}
+	}
+}
+
 // handleDownloadFile handles file download
 func (b *Btelegram) handleDownload(rmsg *config.Message, message *tgbotapi.Message) error {
 	size := 0
@@ -253,15 +304,13 @@ func (b *Btelegram) handleDownload(rmsg *config.Message, message *tgbotapi.Messa
 	if err != nil {
 		return err
 	}
-	if strings.HasSuffix(name, ".webp") && b.GetBool("MediaConvertWebPToPNG") {
-		b.Log.Debugf("WebP to PNG conversion enabled, converting %s", name)
-		err := helper.ConvertWebPToPNG(data)
-		if err != nil {
-			b.Log.Errorf("conversion failed: %s", err)
-		} else {
-			name = strings.Replace(name, ".webp", ".png", 1)
-		}
+
+	if strings.HasSuffix(name, ".tgs.webp") {
+		b.maybeConvertTgs(&name, data)
+	} else if strings.HasSuffix(name, ".webp") {
+		b.maybeConvertWebp(&name, data)
 	}
+
 	helper.HandleDownloadData(b.Log, rmsg, name, message.Caption, "", data, b.General)
 	return nil
 }
@@ -311,6 +360,9 @@ func (b *Btelegram) handleEdit(msg *config.Message, chatid int64) (string, error
 	case "Markdown":
 		b.Log.Debug("Using mode markdown")
 		m.ParseMode = tgbotapi.ModeMarkdown
+	case MarkdownV2:
+		b.Log.Debug("Using mode MarkdownV2")
+		m.ParseMode = MarkdownV2
 	}
 	if strings.ToLower(b.GetString("MessageFormat")) == HTMLNick {
 		b.Log.Debug("Using mode HTML - nick only")
@@ -356,6 +408,14 @@ func (b *Btelegram) handleQuote(message, quoteNick, quoteMessage string) string
 	if format == "" {
 		format = "{MESSAGE} (re @{QUOTENICK}: {QUOTEMESSAGE})"
 	}
+	quoteMessagelength := len(quoteMessage)
+	if b.GetInt("QuoteLengthLimit") != 0 && quoteMessagelength >= b.GetInt("QuoteLengthLimit") {
+		runes := []rune(quoteMessage)
+		quoteMessage = string(runes[0:b.GetInt("QuoteLengthLimit")])
+		if quoteMessagelength > b.GetInt("QuoteLengthLimit") {
+			quoteMessage += "..."
+		}
+	}
 	format = strings.Replace(format, "{MESSAGE}", message, -1)
 	format = strings.Replace(format, "{QUOTENICK}", quoteNick, -1)
 	format = strings.Replace(format, "{QUOTEMESSAGE}", quoteMessage, -1)
@@ -375,8 +435,13 @@ func (b *Btelegram) handleEntities(rmsg *config.Message, message *tgbotapi.Messa
 				b.Log.Errorf("entity text_link url parse failed: %s", err)
 				continue
 			}
-			link := rmsg.Text[e.Offset : e.Offset+e.Length]
-			rmsg.Text = strings.Replace(rmsg.Text, link, url.String(), 1)
+			utfEncodedString := utf16.Encode([]rune(rmsg.Text))
+			if e.Offset+e.Length > len(utfEncodedString) {
+				b.Log.Errorf("entity length is too long %d > %d", e.Offset+e.Length, len(utfEncodedString))
+				continue
+			}
+			link := utf16.Decode(utfEncodedString[e.Offset : e.Offset+e.Length])
+			rmsg.Text = strings.Replace(rmsg.Text, string(link), url.String(), 1)
 		}
 	}
 }

bridge/telegram/telegram.go

@@ -2,19 +2,23 @@ package btelegram
 
 import (
 	"html"
+	"log"
 	"strconv"
 	"strings"
 
 	"github.com/42wim/matterbridge/bridge"
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/42wim/matterbridge/bridge/helper"
-	"github.com/go-telegram-bot-api/telegram-bot-api"
+	tgbotapi "github.com/go-telegram-bot-api/telegram-bot-api"
 )
 
 const (
 	unknownUser = "unknown"
 	HTMLFormat  = "HTML"
 	HTMLNick    = "htmlnick"
+	MarkdownV2  = "MarkdownV2"
+	FormatPng   = "png"
+	FormatWebp  = "webp"
 )
 
 type Btelegram struct {
@@ -24,6 +28,16 @@ type Btelegram struct {
 }
 
 func New(cfg *bridge.Config) bridge.Bridger {
+	tgsConvertFormat := cfg.GetString("MediaConvertTgs")
+	if tgsConvertFormat != "" {
+		err := helper.CanConvertTgsToX()
+		if err != nil {
+			log.Fatalf("Telegram bridge configured to convert .tgs files to '%s', but lottie does not appear to work:\n%#v", tgsConvertFormat, err)
+		}
+		if tgsConvertFormat != FormatPng && tgsConvertFormat != FormatWebp {
+			log.Fatalf("Telegram bridge configured to convert .tgs files to '%s', but only '%s' and '%s' are supported.", FormatPng, FormatWebp, tgsConvertFormat)
+		}
+	}
 	return &Btelegram{Config: cfg, avatarMap: make(map[string]string)}
 }
 
@@ -81,8 +95,8 @@ func (b *Btelegram) Send(msg config.Message) (string, error) {
 	// Upload a file if it exists
 	if msg.Extra != nil {
 		for _, rmsg := range helper.HandleExtra(&msg, b.General) {
-			if _, err := b.sendMessage(chatid, rmsg.Username, rmsg.Text); err != nil {
-				b.Log.Errorf("sendMessage failed: %s", err)
+			if _, msgErr := b.sendMessage(chatid, rmsg.Username, rmsg.Text); msgErr != nil {
+				b.Log.Errorf("sendMessage failed: %s", msgErr)
 			}
 		}
 		// check if we have files to upload (from slack, telegram or mattermost)
@@ -97,7 +111,14 @@ func (b *Btelegram) Send(msg config.Message) (string, error) {
 	}
 
 	// Post normal message
-	return b.sendMessage(chatid, msg.Username, msg.Text)
+	// TODO: recheck it.
+	// Ignore empty text field needs for prevent double messages from whatsapp to telegram
+	// when sending media with text caption
+	if msg.Text != "" {
+		return b.sendMessage(chatid, msg.Username, msg.Text)
+	}
+
+	return "", nil
 }
 
 func (b *Btelegram) getFileDirectURL(id string) string {
@@ -119,11 +140,18 @@ func (b *Btelegram) sendMessage(chatid int64, username, text string) (string, er
 		b.Log.Debug("Using mode markdown")
 		m.ParseMode = tgbotapi.ModeMarkdown
 	}
+	if b.GetString("MessageFormat") == MarkdownV2 {
+		b.Log.Debug("Using mode MarkdownV2")
+		m.ParseMode = MarkdownV2
+	}
 	if strings.ToLower(b.GetString("MessageFormat")) == HTMLNick {
 		b.Log.Debug("Using mode HTML - nick only")
 		m.Text = username + html.EscapeString(text)
 		m.ParseMode = tgbotapi.ModeHTML
 	}
+
+	m.DisableWebPagePreview = b.GetBool("DisableWebPagePreview")
+
 	res, err := b.c.Send(m)
 	if err != nil {
 		return "", err

bridge/whatsapp/handlers.go

@@ -1,14 +1,15 @@
 package bwhatsapp
 
 import (
+	"fmt"
+	"mime"
 	"strings"
 	"time"
 
 	"github.com/42wim/matterbridge/bridge/config"
-
-	"github.com/matterbridge/go-whatsapp"
-
-	whatsappExt "github.com/matterbridge/mautrix-whatsapp/whatsapp-ext"
+	"github.com/42wim/matterbridge/bridge/helper"
+	"github.com/Rhymen/go-whatsapp"
+	"github.com/jpillora/backoff"
 )
 
 /*
@@ -21,7 +22,44 @@ Check:
 
 // HandleError received from WhatsApp
 func (b *Bwhatsapp) HandleError(err error) {
-	b.Log.Errorf("%v", err) // TODO implement proper handling? at least respond to different error types
+	// ignore received invalid data errors. https://github.com/42wim/matterbridge/issues/843
+	// ignore tag 174 errors. https://github.com/42wim/matterbridge/issues/1094
+	if strings.Contains(err.Error(), "error processing data: received invalid data") || strings.Contains(err.Error(), "invalid string with tag 174") {
+		return
+	}
+
+	switch err.(type) {
+	case *whatsapp.ErrConnectionClosed, *whatsapp.ErrConnectionFailed:
+		b.reconnect(err)
+	default:
+		switch err {
+		case whatsapp.ErrConnectionTimeout:
+			b.reconnect(err)
+		default:
+			b.Log.Errorf("%v", err)
+		}
+	}
+}
+
+func (b *Bwhatsapp) reconnect(err error) {
+	bf := &backoff.Backoff{
+		Min:    time.Second,
+		Max:    5 * time.Minute,
+		Jitter: true,
+	}
+	for {
+		d := bf.Duration()
+		b.Log.Errorf("Connection failed, underlying error: %v", err)
+		b.Log.Infof("Waiting %s...", d)
+		time.Sleep(d)
+		b.Log.Info("Reconnecting...")
+		err := b.conn.Restore()
+		if err == nil {
+			bf.Reset()
+			b.startedAt = uint64(time.Now().Unix())
+			return
+		}
+	}
 }
 
 // HandleTextMessage sent from WhatsApp, relay it to the brige
@@ -35,16 +73,18 @@ func (b *Bwhatsapp) HandleTextMessage(message whatsapp.TextMessage) {
 	}
 
 	messageTime := time.Unix(int64(message.Info.Timestamp), 0) // TODO check how behaves between timezones
-	groupJid := message.Info.RemoteJid
+	groupJID := message.Info.RemoteJid
 
-	senderJid := message.Info.SenderJid
-	if len(senderJid) == 0 {
+	senderJID := message.Info.SenderJid
+	if len(senderJID) == 0 {
 		// TODO workaround till https://github.com/Rhymen/go-whatsapp/issues/86 resolved
-		senderJid = *message.Info.Source.Participant
+		if message.Info.Source != nil {
+			senderJID = *message.Info.Source.Participant
+		}
 	}
 
-	// translate sender's Jid to the nicest username we can get
-	senderName := b.getSenderName(senderJid)
+	// translate sender's JID to the nicest username we can get
+	senderName := b.getSenderName(senderJID)
 	if senderName == "" {
 		senderName = "Someone" // don't expose telephone number
 	}
@@ -52,12 +92,12 @@ func (b *Bwhatsapp) HandleTextMessage(message whatsapp.TextMessage) {
 	extText := message.Info.Source.Message.ExtendedTextMessage
 	if extText != nil && extText.ContextInfo != nil && extText.ContextInfo.MentionedJid != nil {
 		// handle user mentions
-		for _, mentionedJid := range extText.ContextInfo.MentionedJid {
-			numberAndSuffix := strings.SplitN(mentionedJid, "@", 2)
+		for _, mentionedJID := range extText.ContextInfo.MentionedJid {
+			numberAndSuffix := strings.SplitN(mentionedJID, "@", 2)
 
 			// mentions comes as telephone numbers and we don't want to expose it to other bridges
 			// replace it with something more meaninful to others
-			mention := b.getSenderNotify(numberAndSuffix[0] + whatsappExt.NewUserSuffix)
+			mention := b.getSenderNotify(numberAndSuffix[0] + "@s.whatsapp.net")
 			if mention == "" {
 				mention = "someone"
 			}
@@ -65,22 +105,22 @@ func (b *Bwhatsapp) HandleTextMessage(message whatsapp.TextMessage) {
 		}
 	}
 
-	b.Log.Debugf("<= Sending message from %s on %s to gateway", senderJid, b.Account)
+	b.Log.Debugf("<= Sending message from %s on %s to gateway", senderJID, b.Account)
 	rmsg := config.Message{
-		UserID:    senderJid,
+		UserID:    senderJID,
 		Username:  senderName,
 		Text:      message.Text,
 		Timestamp: messageTime,
-		Channel:   groupJid,
+		Channel:   groupJID,
 		Account:   b.Account,
 		Protocol:  b.Protocol,
 		Extra:     make(map[string][]interface{}),
-		//		ParentID: TODO, // TODO handle thread replies  // map from Info.QuotedMessageID string
+		//	ParentID: TODO, // TODO handle thread replies  // map from Info.QuotedMessageID string
 		//	Event     string    `json:"event"`
 		//	Gateway   string  // will be added during message processing
 		ID: message.Info.Id}
 
-	if avatarURL, exists := b.userAvatars[senderJid]; exists {
+	if avatarURL, exists := b.userAvatars[senderJID]; exists {
 		rmsg.Avatar = avatarURL
 	}
 
@@ -88,11 +128,75 @@ func (b *Bwhatsapp) HandleTextMessage(message whatsapp.TextMessage) {
 	b.Remote <- rmsg
 }
 
-//
-//func (b *Bwhatsapp) HandleImageMessage(message whatsapp.ImageMessage) {
-//	fmt.Println(message) // TODO implement
-//}
-//
+// HandleImageMessage sent from WhatsApp, relay it to the brige
+func (b *Bwhatsapp) HandleImageMessage(message whatsapp.ImageMessage) {
+	if message.Info.FromMe { // || !strings.Contains(strings.ToLower(message.Text), "@echo") {
+		return
+	}
+
+	// whatsapp sends last messages to show context , cut them
+	if message.Info.Timestamp < b.startedAt {
+		return
+	}
+
+	messageTime := time.Unix(int64(message.Info.Timestamp), 0) // TODO check how behaves between timezones
+	groupJID := message.Info.RemoteJid
+
+	senderJID := message.Info.SenderJid
+	// if len(senderJid) == 0 {
+	//   // TODO workaround till https://github.com/Rhymen/go-whatsapp/issues/86 resolved
+	//   senderJid = *message.Info.Source.Participant
+	// }
+
+	// translate sender's Jid to the nicest username we can get
+	senderName := b.getSenderName(senderJID)
+	if senderName == "" {
+		senderName = "Someone" // don't expose telephone number
+	}
+
+	b.Log.Debugf("<= Sending message from %s on %s to gateway", senderJID, b.Account)
+	rmsg := config.Message{
+		UserID:    senderJID,
+		Username:  senderName,
+		Timestamp: messageTime,
+		Channel:   groupJID,
+		Account:   b.Account,
+		Protocol:  b.Protocol,
+		Extra:     make(map[string][]interface{}),
+		//  ParentID: TODO,      // TODO handle thread replies  // map from Info.QuotedMessageID string
+		//  Event     string    `json:"event"`
+		//  Gateway   string     // will be added during message processing
+		ID: message.Info.Id}
+
+	if avatarURL, exists := b.userAvatars[senderJID]; exists {
+		rmsg.Avatar = avatarURL
+	}
+
+	// Download and unencrypt content
+	data, err := message.Download()
+	if err != nil {
+		b.Log.Errorf("%v", err)
+		return
+	}
+
+	// Get file extension by mimetype
+	fileExt, err := mime.ExtensionsByType(message.Type)
+	if err != nil {
+		b.Log.Errorf("%v", err)
+		return
+	}
+
+	filename := fmt.Sprintf("%v%v", message.Info.Id, fileExt[0])
+
+	b.Log.Debugf("<= Image downloaded and unencrypted")
+
+	// Move file to bridge storage
+	helper.HandleDownloadData(b.Log, &rmsg, filename, message.Caption, "", &data, b.General)
+
+	b.Log.Debugf("<= Image Message is %#v", rmsg)
+	b.Remote <- rmsg
+}
+
 //func (b *Bwhatsapp) HandleVideoMessage(message whatsapp.VideoMessage) {
 //	fmt.Println(message) // TODO implement
 //}

bridge/whatsapp/helpers.go

@@ -2,13 +2,22 @@ package bwhatsapp
 
 import (
 	"encoding/gob"
+	"encoding/json"
 	"errors"
+	"fmt"
 	"os"
 
 	qrcodeTerminal "github.com/Baozisoftware/qrcode-terminal-go"
-	"github.com/matterbridge/go-whatsapp"
+	"github.com/Rhymen/go-whatsapp"
 )
 
+type ProfilePicInfo struct {
+	URL string `json:"eurl"`
+	Tag string `json:"tag"`
+
+	Status int16 `json:"status"`
+}
+
 func qrFromTerminal(invert bool) chan string {
 	qr := make(chan string)
 	go func() {
@@ -71,8 +80,33 @@ func (b *Bwhatsapp) getSenderName(senderJid string) string {
 		// if user is not in phone contacts
 		// it is the most obvious scenario unless you sync your phone contacts with some remote updated source
 		// users can change it in their WhatsApp settings -> profile -> click on Avatar
-		return sender.Notify
+		if sender.Notify != "" {
+			return sender.Notify
+		}
+
+		if sender.Short != "" {
+			return sender.Short
+		}
 	}
+
+	// try to reload this contact
+	_, err := b.conn.Contacts()
+	if err != nil {
+		b.Log.Errorf("error on update of contacts: %v", err)
+	}
+
+	if contact, exists := b.conn.Store.Contacts[senderJid]; exists {
+		// Add it to the user map
+		b.users[senderJid] = contact
+
+		if contact.Name != "" {
+			return contact.Name
+		}
+		// if user is not in phone contacts
+		// same as above
+		return contact.Notify
+	}
+
 	return ""
 }
 
@@ -82,3 +116,17 @@ func (b *Bwhatsapp) getSenderNotify(senderJid string) string {
 	}
 	return ""
 }
+
+func (b *Bwhatsapp) GetProfilePicThumb(jid string) (*ProfilePicInfo, error) {
+	data, err := b.conn.GetProfilePicThumb(jid)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get avatar: %v", err)
+	}
+	content := <-data
+	info := &ProfilePicInfo{}
+	err = json.Unmarshal([]byte(content), info)
+	if err != nil {
+		return info, fmt.Errorf("failed to unmarshal avatar info: %v", err)
+	}
+	return info, nil
+}

bridge/whatsapp/whatsapp.go

@@ -1,20 +1,20 @@
 package bwhatsapp
 
 import (
+	"bytes"
 	"crypto/rand"
 	"encoding/hex"
 	"errors"
 	"fmt"
+	"mime"
 	"os"
+	"path/filepath"
 	"strings"
 	"time"
 
 	"github.com/42wim/matterbridge/bridge"
 	"github.com/42wim/matterbridge/bridge/config"
-
-	"github.com/matterbridge/go-whatsapp"
-
-	whatsappExt "github.com/matterbridge/mautrix-whatsapp/whatsapp-ext"
+	"github.com/Rhymen/go-whatsapp"
 )
 
 const (
@@ -29,10 +29,8 @@ type Bwhatsapp struct {
 	*bridge.Config
 
 	// https://github.com/Rhymen/go-whatsapp/blob/c31092027237441cffba1b9cb148eadf7c83c3d2/session.go#L18-L21
-	session *whatsapp.Session
-	conn    *whatsapp.Conn
-	// https://github.com/tulir/mautrix-whatsapp/blob/master/whatsapp-ext/whatsapp.go
-	connExt   *whatsappExt.ExtendedConn
+	session   *whatsapp.Session
+	conn      *whatsapp.Conn
 	startedAt uint64
 
 	users       map[string]whatsapp.Contact
@@ -69,13 +67,12 @@ func (b *Bwhatsapp) Connect() error {
 	// https://github.com/Rhymen/go-whatsapp#creating-a-connection
 	b.Log.Debugln("Connecting to WhatsApp..")
 	conn, err := whatsapp.NewConn(20 * time.Second)
+	conn.SetClientVersion(0, 4, 2080)
 	if err != nil {
 		return errors.New("failed to connect to WhatsApp: " + err.Error())
 	}
 
 	b.conn = conn
-	b.connExt = whatsappExt.ExtendConn(b.conn)
-	// TODO do we want to use it? b.connExt.SetClientName("Matterbridge WhatsApp bridge", "mb-wa")
 
 	b.conn.AddHandler(b)
 	b.Log.Debugln("WhatsApp connection successful")
@@ -89,7 +86,7 @@ func (b *Bwhatsapp) Connect() error {
 			b.Log.Debugln("Restoring WhatsApp session..")
 
 			// https://github.com/Rhymen/go-whatsapp#restore
-			session, err = b.conn.RestoreSession(session)
+			session, err = b.conn.RestoreWithSession(session)
 			if err != nil {
 				// TODO return or continue to normal login?
 				// restore session connection timed out (I couldn't get over it without logging in again)
@@ -130,7 +127,7 @@ func (b *Bwhatsapp) Connect() error {
 		b.Log.Debug("Getting user avatars..")
 
 		for jid := range b.users {
-			info, err := b.connExt.GetProfilePicThumb(jid)
+			info, err := b.GetProfilePicThumb(jid)
 			if err != nil {
 				b.Log.Warnf("Could not get profile photo of %s: %v", jid, err)
 
@@ -237,6 +234,66 @@ func (b *Bwhatsapp) JoinChannel(channel config.ChannelInfo) error {
 	return nil
 }
 
+// Post a document message from the bridge to WhatsApp
+func (b *Bwhatsapp) PostDocumentMessage(msg config.Message, filetype string) (string, error) {
+	fi := msg.Extra["file"][0].(config.FileInfo)
+
+	// Post document message
+	message := whatsapp.DocumentMessage{
+		Info: whatsapp.MessageInfo{
+			RemoteJid: msg.Channel,
+		},
+		Title:    fi.Name,
+		FileName: fi.Name,
+		Type:     filetype,
+		Content:  bytes.NewReader(*fi.Data),
+	}
+
+	b.Log.Debugf("=> Sending %#v", msg)
+
+	// create message ID
+	// TODO follow and act if https://github.com/Rhymen/go-whatsapp/issues/101 implemented
+	idBytes := make([]byte, 10)
+	if _, err := rand.Read(idBytes); err != nil {
+		b.Log.Warn(err.Error())
+	}
+
+	message.Info.Id = strings.ToUpper(hex.EncodeToString(idBytes))
+	_, err := b.conn.Send(message)
+
+	return message.Info.Id, err
+}
+
+// Post an image message from the bridge to WhatsApp
+// Handle, for sure image/jpeg, image/png and image/gif MIME types
+func (b *Bwhatsapp) PostImageMessage(msg config.Message, filetype string) (string, error) {
+	fi := msg.Extra["file"][0].(config.FileInfo)
+
+	// Post image message
+	message := whatsapp.ImageMessage{
+		Info: whatsapp.MessageInfo{
+			RemoteJid: msg.Channel,
+		},
+		Type:    filetype,
+		Caption: msg.Username + fi.Comment,
+		Content: bytes.NewReader(*fi.Data),
+	}
+
+	b.Log.Debugf("=> Sending %#v", msg)
+
+	// create message ID
+	// TODO follow and act if https://github.com/Rhymen/go-whatsapp/issues/101 implemented
+	idBytes := make([]byte, 10)
+	if _, err := rand.Read(idBytes); err != nil {
+		b.Log.Warn(err.Error())
+	}
+
+	message.Info.Id = strings.ToUpper(hex.EncodeToString(idBytes))
+	_, err := b.conn.Send(message)
+
+	return message.Info.Id, err
+}
+
 // Send a message from the bridge to WhatsApp
 // Required implementation of the Bridger interface
 // https://github.com/42wim/matterbridge/blob/2cfd880cdb0df29771bf8f31df8d990ab897889d/bridge/bridge.go#L11-L16
@@ -266,18 +323,25 @@ func (b *Bwhatsapp) Send(msg config.Message) (string, error) {
 		// TODO handle edit as a message reply with updated text
 	}
 
-	//// TODO Handle Upload a file
-	//if msg.Extra != nil {
-	//	for _, rmsg := range helper.HandleExtra(&msg, b.General) {
-	//		b.c.SendMessage(roomID, rmsg.Username+rmsg.Text)
-	//	}
-	//	if len(msg.Extra["file"]) > 0 {
-	//		return b.handleUploadFile(&msg, roomID)
-	//	}
-	//}
+	// Handle Upload a file
+	if msg.Extra["file"] != nil {
+		fi := msg.Extra["file"][0].(config.FileInfo)
+		filetype := mime.TypeByExtension(filepath.Ext(fi.Name))
+
+		b.Log.Debugf("Extra file is %#v", filetype)
+
+		// TODO: add different types
+		// TODO: add webp conversion
+		switch filetype {
+		case "image/jpeg", "image/png", "image/gif":
+			return b.PostImageMessage(msg, filetype)
+		default:
+			return b.PostDocumentMessage(msg, filetype)
+		}
+	}
 
 	// Post text message
-	text := whatsapp.TextMessage{
+	message := whatsapp.TextMessage{
 		Info: whatsapp.MessageInfo{
 			RemoteJid: msg.Channel, // which equals to group id
 		},
@@ -288,15 +352,14 @@ func (b *Bwhatsapp) Send(msg config.Message) (string, error) {
 
 	// create message ID
 	// TODO follow and act if https://github.com/Rhymen/go-whatsapp/issues/101 implemented
-	bytes := make([]byte, 10)
-	if _, err := rand.Read(bytes); err != nil {
+	idBytes := make([]byte, 10)
+	if _, err := rand.Read(idBytes); err != nil {
 		b.Log.Warn(err.Error())
 	}
-	text.Info.Id = strings.ToUpper(hex.EncodeToString(bytes))
+	message.Info.Id = strings.ToUpper(hex.EncodeToString(idBytes))
+	_, err := b.conn.Send(message)
 
-	err := b.conn.Send(text)
-
-	return text.Info.Id, err
+	return message.Info.Id, err
 }
 
 // TODO do we want that? to allow login with QR code from a bridged channel? https://github.com/tulir/mautrix-whatsapp/blob/513eb18e2d59bada0dd515ee1abaaf38a3bfe3d5/commands.go#L76

bridge/xmpp/handler.go

@@ -0,0 +1,34 @@
+package bxmpp
+
+import (
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/42wim/matterbridge/bridge/helper"
+	"github.com/matterbridge/go-xmpp"
+)
+
+// handleDownloadAvatar downloads the avatar of userid from channel
+// sends a EVENT_AVATAR_DOWNLOAD message to the gateway if successful.
+// logs an error message if it fails
+func (b *Bxmpp) handleDownloadAvatar(avatar xmpp.AvatarData) {
+	rmsg := config.Message{
+		Username: "system",
+		Text:     "avatar",
+		Channel:  b.parseChannel(avatar.From),
+		Account:  b.Account,
+		UserID:   avatar.From,
+		Event:    config.EventAvatarDownload,
+		Extra:    make(map[string][]interface{}),
+	}
+	if _, ok := b.avatarMap[avatar.From]; !ok {
+		b.Log.Debugf("Avatar.From: %s", avatar.From)
+
+		err := helper.HandleDownloadSize(b.Log, &rmsg, avatar.From+".png", int64(len(avatar.Data)), b.General)
+		if err != nil {
+			b.Log.Error(err)
+			return
+		}
+		helper.HandleDownloadData(b.Log, &rmsg, avatar.From+".png", rmsg.Text, "", &avatar.Data, b.General)
+		b.Log.Debugf("Avatar download complete")
+		b.Remote <- rmsg
+	}
+}

bridge/xmpp/helpers.go

@@ -0,0 +1,30 @@
+package bxmpp
+
+import (
+	"regexp"
+
+	"github.com/42wim/matterbridge/bridge/config"
+)
+
+var pathRegex = regexp.MustCompile("[^a-zA-Z0-9]+")
+
+// GetAvatar constructs a URL for a given user-avatar if it is available in the cache.
+func getAvatar(av map[string]string, userid string, general *config.Protocol) string {
+	if hash, ok := av[userid]; ok {
+		// NOTE: This does not happen in bridge/helper/helper.go but messes up XMPP
+		id := pathRegex.ReplaceAllString(userid, "_")
+		return general.MediaServerDownload + "/" + hash + "/" + id + ".png"
+	}
+	return ""
+}
+
+func (b *Bxmpp) cacheAvatar(msg *config.Message) string {
+	fi := msg.Extra["file"][0].(config.FileInfo)
+	/* if we have a sha we have successfully uploaded the file to the media server,
+	so we can now cache the sha */
+	if fi.SHA != "" {
+		b.Log.Debugf("Added %s to %s in avatarMap", fi.SHA, msg.UserID)
+		b.avatarMap[msg.UserID] = fi.SHA
+	}
+	return ""
+}

bridge/xmpp/xmpp.go

@@ -2,7 +2,9 @@ package bxmpp
 
 import (
 	"crypto/tls"
+	"fmt"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/42wim/matterbridge/bridge"
@@ -14,50 +16,36 @@ import (
 )
 
 type Bxmpp struct {
-	xc      *xmpp.Client
-	xmppMap map[string]string
 	*bridge.Config
+
 	startTime time.Time
+	xc        *xmpp.Client
+	xmppMap   map[string]string
+	connected bool
+	sync.RWMutex
+
+	avatarAvailability map[string]bool
+	avatarMap          map[string]string
 }
 
 func New(cfg *bridge.Config) bridge.Bridger {
-	b := &Bxmpp{Config: cfg}
-	b.xmppMap = make(map[string]string)
-	return b
+	return &Bxmpp{
+		Config:             cfg,
+		xmppMap:            make(map[string]string),
+		avatarAvailability: make(map[string]bool),
+		avatarMap:          make(map[string]string),
+	}
 }
 
 func (b *Bxmpp) Connect() error {
-	var err error
 	b.Log.Infof("Connecting %s", b.GetString("Server"))
-	b.xc, err = b.createXMPP()
-	if err != nil {
+	if err := b.createXMPP(); err != nil {
 		b.Log.Debugf("%#v", err)
 		return err
 	}
+
 	b.Log.Info("Connection succeeded")
-	go func() {
-		initial := true
-		bf := &backoff.Backoff{
-			Min:    time.Second,
-			Max:    5 * time.Minute,
-			Jitter: true,
-		}
-		for {
-			if initial {
-				b.handleXMPP()
-				initial = false
-			}
-			d := bf.Duration()
-			b.Log.Infof("Disconnected. Reconnecting in %s", d)
-			time.Sleep(d)
-			b.xc, err = b.createXMPP()
-			if err == nil {
-				b.Remote <- config.Message{Username: "system", Text: "rejoin", Channel: "", Account: b.Account, Event: config.EventRejoinChannels}
-				b.handleXMPP()
-				bf.Reset()
-			}
-		}
-	}()
+	go b.manageConnection()
 	return nil
 }
 
@@ -76,40 +64,75 @@ func (b *Bxmpp) JoinChannel(channel config.ChannelInfo) error {
 }
 
 func (b *Bxmpp) Send(msg config.Message) (string, error) {
+	// should be fixed by using a cache instead of dropping
+	if !b.Connected() {
+		return "", fmt.Errorf("bridge %s not connected, dropping message %#v to bridge", b.Account, msg)
+	}
 	// ignore delete messages
 	if msg.Event == config.EventMsgDelete {
 		return "", nil
 	}
+
 	b.Log.Debugf("=> Receiving %#v", msg)
 
-	// Upload a file (in xmpp case send the upload URL because xmpp has no native upload support)
+	if msg.Event == config.EventAvatarDownload {
+		return b.cacheAvatar(&msg), nil
+	}
+
+	// Make a action /me of the message, prepend the username with it.
+	// https://xmpp.org/extensions/xep-0245.html
+	if msg.Event == config.EventUserAction {
+		msg.Username = "/me " + msg.Username
+	}
+
+	// Upload a file (in XMPP case send the upload URL because XMPP has no native upload support).
 	if msg.Extra != nil {
 		for _, rmsg := range helper.HandleExtra(&msg, b.General) {
-			b.xc.Send(xmpp.Chat{Type: "groupchat", Remote: rmsg.Channel + "@" + b.GetString("Muc"), Text: rmsg.Username + rmsg.Text})
+			b.Log.Debugf("=> Sending attachement message %#v", rmsg)
+			if _, err := b.xc.Send(xmpp.Chat{
+				Type:   "groupchat",
+				Remote: rmsg.Channel + "@" + b.GetString("Muc"),
+				Text:   rmsg.Username + rmsg.Text,
+			}); err != nil {
+				b.Log.WithError(err).Error("Unable to send message with share URL.")
+			}
 		}
 		if len(msg.Extra["file"]) > 0 {
-			return b.handleUploadFile(&msg)
+			return "", b.handleUploadFile(&msg)
 		}
 	}
 
-	var msgreplaceid string
-	msgid := xid.New().String()
+	var msgReplaceID string
+	msgID := xid.New().String()
 	if msg.ID != "" {
-		msgid = msg.ID
-		msgreplaceid = msg.ID
+		msgID = msg.ID
+		msgReplaceID = msg.ID
 	}
-	// Post normal message
-	_, err := b.xc.Send(xmpp.Chat{Type: "groupchat", Remote: msg.Channel + "@" + b.GetString("Muc"), Text: msg.Username + msg.Text, ID: msgid, ReplaceID: msgreplaceid})
-	if err != nil {
+	// Post normal message.
+	b.Log.Debugf("=> Sending message %#v", msg)
+	if _, err := b.xc.Send(xmpp.Chat{
+		Type:      "groupchat",
+		Remote:    msg.Channel + "@" + b.GetString("Muc"),
+		Text:      msg.Username + msg.Text,
+		ID:        msgID,
+		ReplaceID: msgReplaceID,
+	}); err != nil {
 		return "", err
 	}
-	return msgid, nil
+	return msgID, nil
 }
 
-func (b *Bxmpp) createXMPP() (*xmpp.Client, error) {
-	tc := new(tls.Config)
-	tc.InsecureSkipVerify = b.GetBool("SkipTLSVerify")
-	tc.ServerName = strings.Split(b.GetString("Server"), ":")[0]
+func (b *Bxmpp) createXMPP() error {
+	if !strings.Contains(b.GetString("Jid"), "@") {
+		return fmt.Errorf("the Jid %s doesn't contain an @", b.GetString("Jid"))
+	}
+	tc := &tls.Config{
+		ServerName:         strings.Split(b.GetString("Jid"), "@")[1],
+		InsecureSkipVerify: b.GetBool("SkipTLSVerify"), // nolint: gosec
+	}
+
+	xmpp.DebugWriter = b.Log.Writer()
+
 	options := xmpp.Options{
 		Host:                         b.GetString("Server"),
 		User:                         b.GetString("Jid"),
@@ -118,7 +141,6 @@ func (b *Bxmpp) createXMPP() (*xmpp.Client, error) {
 		StartTLS:                     true,
 		TLSConfig:                    tc,
 		Debug:                        b.GetBool("debug"),
-		Logger:                       b.Log.Writer(),
 		Session:                      true,
 		Status:                       "",
 		StatusMessage:                "",
@@ -127,7 +149,54 @@ func (b *Bxmpp) createXMPP() (*xmpp.Client, error) {
 	}
 	var err error
 	b.xc, err = options.NewClient()
-	return b.xc, err
+	return err
+}
+
+func (b *Bxmpp) manageConnection() {
+	b.setConnected(true)
+	initial := true
+	bf := &backoff.Backoff{
+		Min:    time.Second,
+		Max:    5 * time.Minute,
+		Jitter: true,
+	}
+
+	// Main connection loop. Each iteration corresponds to a successful
+	// connection attempt and the subsequent handling of the connection.
+	for {
+		if initial {
+			initial = false
+		} else {
+			b.Remote <- config.Message{
+				Username: "system",
+				Text:     "rejoin",
+				Channel:  "",
+				Account:  b.Account,
+				Event:    config.EventRejoinChannels,
+			}
+		}
+
+		if err := b.handleXMPP(); err != nil {
+			b.Log.WithError(err).Error("Disconnected.")
+			b.setConnected(false)
+		}
+
+		// Reconnection loop using an exponential back-off strategy. We
+		// only break out of the loop if we have successfully reconnected.
+		for {
+			d := bf.Duration()
+			b.Log.Infof("Reconnecting in %s.", d)
+			time.Sleep(d)
+
+			b.Log.Infof("Reconnecting now.")
+			if err := b.createXMPP(); err == nil {
+				b.setConnected(true)
+				bf.Reset()
+				break
+			}
+			b.Log.Warn("Failed to reconnect.")
+		}
+	}
 }
 
 func (b *Bxmpp) xmppKeepAlive() chan bool {
@@ -139,8 +208,7 @@ func (b *Bxmpp) xmppKeepAlive() chan bool {
 			select {
 			case <-ticker.C:
 				b.Log.Debugf("PING")
-				err := b.xc.PingC2S("", "")
-				if err != nil {
+				if err := b.xc.PingC2S("", ""); err != nil {
 					b.Log.Debugf("PING failed %#v", err)
 				}
 			case <-done:
@@ -152,53 +220,74 @@ func (b *Bxmpp) xmppKeepAlive() chan bool {
 }
 
 func (b *Bxmpp) handleXMPP() error {
-	var ok bool
-	var msgid string
 	b.startTime = time.Now()
+
 	done := b.xmppKeepAlive()
 	defer close(done)
+
 	for {
 		m, err := b.xc.Recv()
 		if err != nil {
 			return err
 		}
+
 		switch v := m.(type) {
 		case xmpp.Chat:
 			if v.Type == "groupchat" {
 				b.Log.Debugf("== Receiving %#v", v)
-				event := ""
-				// skip invalid messages
+
+				// Skip invalid messages.
 				if b.skipMessage(v) {
 					continue
 				}
+
+				var event string
 				if strings.Contains(v.Text, "has set the subject to:") {
 					event = config.EventTopicChange
 				}
-				msgid = v.ID
+
+				available, sok := b.avatarAvailability[v.Remote]
+				avatar := ""
+				if !sok {
+					b.Log.Debugf("Requesting avatar data")
+					b.avatarAvailability[v.Remote] = false
+					b.xc.AvatarRequestData(v.Remote)
+				} else if available {
+					avatar = getAvatar(b.avatarMap, v.Remote, b.General)
+				}
+
+				msgID := v.ID
 				if v.ReplaceID != "" {
-					msgid = v.ReplaceID
+					msgID = v.ReplaceID
 				}
 				rmsg := config.Message{
 					Username: b.parseNick(v.Remote),
 					Text:     v.Text,
 					Channel:  b.parseChannel(v.Remote),
 					Account:  b.Account,
+					Avatar:   avatar,
 					UserID:   v.Remote,
-					ID:       msgid,
+					ID:       msgID,
 					Event:    event,
 				}
 
-				// check if we have an action event
+				// Check if we have an action event.
+				var ok bool
 				rmsg.Text, ok = b.replaceAction(rmsg.Text)
 				if ok {
 					rmsg.Event = config.EventUserAction
 				}
+
 				b.Log.Debugf("<= Sending message from %s on %s to gateway", rmsg.Username, b.Account)
 				b.Log.Debugf("<= Message is %#v", rmsg)
 				b.Remote <- rmsg
 			}
+		case xmpp.AvatarData:
+			b.handleDownloadAvatar(v)
+			b.avatarAvailability[v.From] = true
+			b.Log.Debugf("Avatar for %s is now available", v.From)
 		case xmpp.Presence:
-			// do nothing
+			// Do nothing.
 		}
 	}
 }
@@ -211,30 +300,41 @@ func (b *Bxmpp) replaceAction(text string) (string, bool) {
 }
 
 // handleUploadFile handles native upload of files
-func (b *Bxmpp) handleUploadFile(msg *config.Message) (string, error) {
-	var urldesc = ""
+func (b *Bxmpp) handleUploadFile(msg *config.Message) error {
+	var urlDesc string
 
-	for _, f := range msg.Extra["file"] {
-		fi := f.(config.FileInfo)
-		if fi.Comment != "" {
-			msg.Text += fi.Comment + ": "
+	for _, file := range msg.Extra["file"] {
+		fileInfo := file.(config.FileInfo)
+		if fileInfo.Comment != "" {
+			msg.Text += fileInfo.Comment + ": "
 		}
-		if fi.URL != "" {
-			msg.Text = fi.URL
-			if fi.Comment != "" {
-				msg.Text = fi.Comment + ": " + fi.URL
-				urldesc = fi.Comment
+		if fileInfo.URL != "" {
+			msg.Text = fileInfo.URL
+			if fileInfo.Comment != "" {
+				msg.Text = fileInfo.Comment + ": " + fileInfo.URL
+				urlDesc = fileInfo.Comment
 			}
 		}
-		_, err := b.xc.Send(xmpp.Chat{Type: "groupchat", Remote: msg.Channel + "@" + b.GetString("Muc"), Text: msg.Username + msg.Text})
-		if err != nil {
-			return "", err
+		if _, err := b.xc.Send(xmpp.Chat{
+			Type:   "groupchat",
+			Remote: msg.Channel + "@" + b.GetString("Muc"),
+			Text:   msg.Username + msg.Text,
+		}); err != nil {
+			return err
 		}
-		if fi.URL != "" {
-			b.xc.SendOOB(xmpp.Chat{Type: "groupchat", Remote: msg.Channel + "@" + b.GetString("Muc"), Ooburl: fi.URL, Oobdesc: urldesc})
+
+		if fileInfo.URL != "" {
+			if _, err := b.xc.SendOOB(xmpp.Chat{
+				Type:    "groupchat",
+				Remote:  msg.Channel + "@" + b.GetString("Muc"),
+				Ooburl:  fileInfo.URL,
+				Oobdesc: urlDesc,
+			}); err != nil {
+				b.Log.WithError(err).Warn("Failed to send share URL.")
+			}
 		}
 	}
-	return "", nil
+	return nil
 }
 
 func (b *Bxmpp) parseNick(remote string) string {
@@ -279,6 +379,17 @@ func (b *Bxmpp) skipMessage(message xmpp.Chat) bool {
 	}
 
 	// skip delayed messages
-	t := time.Time{}
-	return message.Stamp != t
+	return !message.Stamp.IsZero() && time.Since(message.Stamp).Minutes() > 5
+}
+
+func (b *Bxmpp) setConnected(state bool) {
+	b.Lock()
+	b.connected = state
+	defer b.Unlock()
+}
+
+func (b *Bxmpp) Connected() bool {
+	b.RLock()
+	defer b.RUnlock()
+	return b.connected
 }

changelog.md

@@ -1,3 +1,332 @@
+# v1.18.1
+
+## New features
+
+- telegram: Support Telegram animated stickers (tgs) format (#1173). See https://github.com/42wim/matterbridge/wiki/Settings#mediaConverttgs for more info
+
+## Enhancements
+
+- matrix: Remove HTML formatting for push messages (#1188) (#1189)
+- mattermost: Use mattermost v5 module (#1192)
+
+## Bugfix
+
+- whatsapp: Handle panic in whatsapp. Fixes #1180 (#1184)
+- nctalk: Fix Nextcloud Talk connection failure (#1179)
+- matrix: Sleep when ratelimited on joins (matrix). Fixes #1201 (#1206)
+
+This release couldn't exist without the following contributors:
+@42wim, @BenWiederhake, @Dellle, @gary-kim
+
+# v1.18.0
+
+## New features
+
+- nctalk: new protocol added. Add Nextcloud Talk support #1167
+- general: Add an option to log into a file rather than stdout (#1168)
+- api: Add websocket to API (#970)
+
+## Enhancements
+
+- telegram: Fix MarkdownV2 support in Telegram (#1169)
+- whatsapp: Reload user information when a new contact is detected (whatsapp) (#1160)
+- api: Add sane RemoteNickFormat default for API (#1157)
+- irc: Skip gIRC built-in rate limiting (irc) (#1164)
+- irc: Only colour IRC nicks if there is one. (#1161)
+- docker: Combine runs to one layer (#1151)
+
+## Bugfix
+
+- general: Update dependencies for 1.18.0 release (#1175)
+
+Discord users are encouraged to upgrade, this release works with the move to the discord.com domain.
+
+This release couldn't exist without the following contributors:
+@42wim, @jlu5, @qaisjp, @TheHolyRoger, @SuperSandro2000, @gary-kim, @z3bra, @greenx, @haykam821, @nathanaelhoun
+
+# v1.17.5
+
+## Enhancements
+
+- irc: Add StripMarkdown option (irc). (#1145)
+- general: Increase debug logging with function,file and linenumber (#1147)
+- general: Update Dockerfile so inotify works (#1148)
+- matrix: Add an option to disable sending HTML to matrix. Fixes #1022 (#1135)
+- xmpp: Implement xep-0245 (xmpp). Closes #1137 (#1144)
+
+## Bugfix
+
+- discord: Fix #1120: replaceAction "_" crash (discord) (#1121)
+- discord: Fix #1049: missing space before embeds (discord) (#1124)
+- discord: Fix webhook EventUserAction messages being skipped (discord) (#1133)
+- matrix: Avoid creating invalid url when the user doesn't have an avatar (matrix) (#1130)
+- msteams: Ignore non-user messages (msteams). Fixes #1141 (#1149)
+- slack: Do not use webhooks when token is configured (slack) (fixes #1123) (#1134)
+- telegram: Fix forward from hidden users (telegram). Closes #1131 (#1143)
+- xmpp: Prevent re-requesting avatar data (xmpp) (#1117)
+
+This release couldn't exist without the following contributors:
+@qaisjp, @xnaas, @42wim, @Polynomdivision, @tfve
+
+# v1.17.4
+
+## Bugfix
+
+- general: Lowercase account names. Fixes #1108 (#1110)
+- msteams: Remove panics and retry polling on failure (msteams). Fixes #1104 (#1105
+- whatsapp: Update Rhymen/go-whatsapp. Fixes #1107 (#1109) (make whatsapp working again)
+- discord: Add an ID cache (discord). Fixes #1106 (#1111) (fix delete/edits with webhooks)
+
+# v1.17.3
+
+## Enhancements
+
+- xmpp: Implement User Avatar spoofing of XMPP users #1090
+- rocketchat: Relay Joins/Topic changes in RocketChat bridge (#1085)
+- irc: Add JoinDelay option (irc). Fixes #1084 (#1098)
+- slack: Clip too long messages on 3000 length (slack). Fixes #1081 (#1102)
+
+## Bugfix
+
+- general: Fix the behavior of ShowTopicChange and SyncTopic (#1086)
+- slack: Prevent image/message looping (slack). Fixes #1088 (#1096)
+- whatsapp: Ignore non-critical errors (whatsapp). Fixes #1094 (#1100)
+- irc: Add extra space before colon in attachments (irc). Fixes #1089 (#1101)
+
+This release couldn't exist without the following contributors:
+@42wim, @ldruschk, @qaisjp, @Polynomdivision
+
+# v1.17.2
+
+## Enhancements
+
+- slack: Update vendor slack-go/slack (#1068)
+- general: Update vendor d5/tengo (#1066)
+- general: Clarify terminology used in mapping group chat IDs to channels in config (#1079)
+
+## Bugfix
+
+- whatsapp: Update Rhymen/go-whatsapp vendor and whatsapp version (#1078). Fixes Media upload #1074
+- whatsapp: Reset start timestamp on reconnect (whatsapp). Fixes #1059 (#1064)
+
+This release couldn't exist without the following contributors:
+@42wim, @jheiselman
+
+# v1.17.1
+
+## Enhancements
+
+- docker: Remove build dependencies from final image (multistage build) #1057
+
+## Bugfix
+
+- general: Don't transmit typing events from ourselves #1056
+- general: Add support for build tags #1054
+- discord: Strip extra info from emotes (discord) #1052
+- msteams: fix macos build: Update vendor yaegashi/msgraph.go to v0.1.2 #1036
+- whatsapp: Update client version whatsapp. Fixes #1061 #1062
+
+This release couldn't exist without the following contributors:
+@awigen, @qaisjp, @42wim
+
+# v1.17.0
+
+## New features
+
+- msteams: new protocol added. Add initial Microsoft Teams support #967
+  See https://github.com/42wim/matterbridge/wiki/MS-Teams-setup for a complete walkthrough
+- discord: Add ability to procure avatars from the destination bridge #1000
+- matrix: Add support for avatars from matrix. #1007
+- general: support JSON and YAML config formats #1045
+
+## Enhancements
+
+- discord: Check only bridged channels for PermManageWebhooks #1001
+- irc: Be less lossy when throttling IRC messages #1004
+- keybase: updated library #1002, #1019
+- matrix: Rebase gomatrix vendor with upstream #1006
+- slack: Use upstream slack-go/slack again #1018
+- slack: Ignore ConnectingEvent #1041
+- slack: use blocks not attachments #1048
+- sshchat: Update vendor shazow/ssh-chat #1029
+- telegram: added markdownv2 mode for telegram #1037
+- whatsapp: Implement basic reconnect (whatsapp). Fixes #987 #1003
+
+## Bugfix
+
+- discord: Fix webhook permission checks sometimes failing #1043
+- discord: Fix #1027: warning when handling inbound webhooks #1044
+- discord: Fix duplicate separator on empty description/url (discord) #1035
+- matrix: Fix issue with underscores in links #999
+- slack: Fix #1039: messages sent to Slack being synced back #1046
+- telegram: Make avatars download work with mediaserverdownload (telegram). Fixes #920 
+
+This release couldn't exist without the following contributors:
+@qaisjp, @jakubgs, @burner1024, @notpushkin, @MartijnBraam, @42wim
+
+# v1.16.5
+
+- Fix version bump
+
+# v1.16.4
+
+## New features
+
+- whatsapp: Add support for WhatsApp media (jpeg/png/gif) bridging (#974)
+- telegram: Add QuoteLengthLimit option (telegram) fixes #963 (#985)
+- telegram: Add DisableWebPagePreview option (telegram). Closes #980 (#994)
+
+## Enhancements
+
+- general: update dependencies
+- tengo: update to tengo v2
+- general: Add Docker Compose configuration (#990)
+
+## Bugfix
+
+- general: Fail with message instead of panic. #988 (#991)
+- telegram: Add extra mimetypes to docker image. Fixes #969
+- discord: Fix channel ID problem with multiple gateways (discord). Fixes #953 (#977)
+- discord: Show file comment in webhook if normal message is empty (discord). Fixes #962 (#995)
+- matrix: Fix parsing issues - Disable smartypants in markdown parser. Fixes #989, #983 (#993)
+- sshchat: Fix duplicated messages (sshchat). Fixes #950 (#996)
+
+This release couldn't exist without the following contributors:
+@jwflory, @42wim, @pbek, @Humorhenker, @c0ncord2, @glazzara
+
+# v1.16.3
+
+## Bugfix
+
+- slack: Fix issues with ratelimiting #959
+- mattermost: Fix bug when using webhookURL and login/token together #960
+
+# v1.16.2
+
+## New features
+
+- keybase: Add support for receiving attachments (keybase) (#923)
+
+## Enhancements
+
+- general: Switch to new emoji library kyokomi/emoji (#948)
+- general: Update markdown parsing library to github.com/gomarkdown/markdown (#944)
+- ssh-chat: Update shazow/ssh-chat dependency (#947)
+
+## Bugfix
+
+- slack: Fix issues with the slack block kit API #937 (#943).
+
+This release couldn't exist without the following contributors:
+@42wim, @bmpickford, @goncalor
+
+# v1.16.1
+
+## New features
+
+* rocketchat: add token support #892
+* matrix: Add support for uploading application/x and audio/x (matrix). #929
+
+## Enhancements
+
+* general: Do configuration validation on start-up. Fixes #888
+* general: updated vendored libraries (discord/whatsapp) #932
+* discord: user typing messages #914
+* slack: Convert slack bold/strike to correct markdown (slack). Fixes #918
+
+## Bugfix
+
+* discord: fix Failed to fetch information for members message. #894
+* discord: remove obsolete file upload links (discord). #931
+* slack: suppress unhandled HelloEvent message #913
+* mattermost: Fix panic on WebhookURL only setting (mattermost). #917
+* matrix: fix corrupted links between slack and matrix #924
+
+This release couldn't exist without the following contributors:
+@qaisjp, @hramrach, @42wim
+
+# v1.16.0
+
+## New features
+
+* keybase: new protocol added. Add initial Keybase Chat support #877 Thanks to @hyperobject
+* discord: Support webhook files in discord #872
+
+## Enhancements
+
+* general: update dependencies
+
+## Bugfix
+
+* discord: Underscores from Discord don't arrive correctly #864
+* xmpp: Fix possible panic at startup of the XMPP bridge #869
+* mattermost: Make getChannelIdTeam behave like GetChannelId for groups (mattermost) #873
+
+This release couldn't exist without the following contributors:
+@hyperobject, @42wim, @bucko909, @MOZGIII
+
+# v1.15.1
+
+## New features
+* discord: Support webhook message deletions (discord) (#853)
+
+## Enhancements
+
+* discord: Support bulk deletions #851
+* discord: Support channels in categories #863 (use category/channel. See matterbridge.toml.sample for more info)
+* mattermost: Add an option to skip the Mattermost server version check #849
+
+## Bugfix
+
+* xmpp: fix segfault when disconnected/reconnected #856
+* telegram: fix panic in handleEntities #858
+
+This release couldn't exist without the following contributors:
+@42wim, @qaisjp, @joohoi
+
+# v1.15.0
+## New features
+* Add scripting (tengo) support for every outgoing message (#806)
+  See https://github.com/42wim/matterbridge/wiki/Settings#tengo and 
+  https://github.com/42wim/matterbridge/wiki/Settings#outmessage for more information
+* Add tengo support to RemoteNickFormat (#793)
+  See https://github.com/42wim/matterbridge/wiki/Settings#remotenickformat-2
+  * Deprecated `Message` under `[tengo]` to `InMessage`
+
+## Enhancements
+  * general: Forward only user-typing messages if supported by protocol (#832)
+  * general: updated wiki with all possible settings: https://github.com/42wim/matterbridge/wiki/Settings
+  * tengo: Add msg event to tengo
+  * xmpp: Verify TLS against JID domain, not the host. (xmpp) (#834)
+  * xmpp: Allow messages with timestamp (xmpp). Fixes #835 (#847)
+  * irc: Add verbose IRC joins/parts (ident@host) (#805)
+  See https://github.com/42wim/matterbridge/wiki/Settings#verbosejoinpart
+  * rocketchat: Add useraction support (rocketchat). Closes #772 (#794)
+
+## Bugfix
+  * slack: Fix regression in autojoining with legacy tokens (slack). Fixes #651 (#848)
+  * xmpp: Revert xmpp to orig behaviour. Closes #844
+* whatsapp: Update github.com/Rhymen/go-whatsapp vendor. Fixes #843
+* mattermost: Update channels of all teams (mattermost)
+
+This release couldn't exist without the following contributors:
+@42wim, @Helcaraxan, @chotaire, @qaisjp, @dajohi, @kousu
+
+# v1.14.4
+
+## Bugfix
+* mattermost: Add Id to EditMessage (mattermost). Fixes #802
+* mattermost: Fix panic on nil message.Post (mattermost). Fixes #804
+* mattermost: Handle unthreaded messages (mattermost). Fixes #803
+* mattermost: Use paging in initUser and UpdateUsers (mattermost)
+* slack: Add lacking clean-up in Slack synchronisation (#811)
+* slack: Disable user lookups on delete messages (slack) (#812)
+
+# v1.14.3
+
+## Bugfix
+* irc: Fix deadlock on reconnect (irc). Closes #757
+
 # v1.14.2
 
 ## Bugfix

ci/bintray.sh

@@ -1,30 +0,0 @@
-#!/usr/bin/env bash
-set -u -e -x -o pipefail
-
-go version | grep go1.12 || exit
-
-VERSION=$(git describe --tags)
-mkdir ci/binaries
-GOOS=windows GOARCH=amd64 go build -ldflags "-s -w -X main.githash=$(git log --pretty=format:'%h' -n 1)" -o ci/binaries/matterbridge-$VERSION-windows-amd64.exe
-GOOS=linux GOARCH=amd64 go build -ldflags "-s -w -X main.githash=$(git log --pretty=format:'%h' -n 1)" -o ci/binaries/matterbridge-$VERSION-linux-amd64
-GOOS=linux GOARCH=arm go build -ldflags "-s -w -X main.githash=$(git log --pretty=format:'%h' -n 1)" -o ci/binaries/matterbridge-$VERSION-linux-arm
-GOOS=darwin GOARCH=amd64 go build -ldflags "-s -w -X main.githash=$(git log --pretty=format:'%h' -n 1)" -o ci/binaries/matterbridge-$VERSION-darwin-amd64
-cd ci
-cat > deploy.json <<EOF
-{
-    "package": {
-        "name": "Matterbridge",
-        "repo": "nightly",
-        "subject": "42wim"
-    },
-    "version": {
-        "name": "$VERSION"
-    },
-    "files":
-        [
-        {"includePattern": "ci/binaries/(.*)", "uploadPattern":"\$1"}
-        ],
-    "publish": true
-}
-EOF
-

ci/lint.sh

@@ -1,17 +0,0 @@
-#!/usr/bin/env bash
-set -u -e -x -o pipefail
-
-if [[ -n "${GOLANGCI_VERSION-}" ]]; then
-  # Retrieve the golangci-lint linter binary.
-  curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | bash -s -- -b ${GOPATH}/bin ${GOLANGCI_VERSION}
-fi
-
-# Run the linter.
-golangci-lint run
-
-if [[ "${GO111MODULE-off}" == "on" ]]; then
-  # If Go modules are active then check that dependencies are correctly maintained.
-  go mod tidy
-  go mod vendor
-  git diff --exit-code --quiet || (echo "Please run 'go mod tidy' to clean up the 'go.mod' and 'go.sum' files."; false)
-fi

ci/test.sh

@@ -1,17 +0,0 @@
-#!/usr/bin/env bash
-set -u -e -x -o pipefail
-
-if [[ -n "${REPORT_COVERAGE+cover}" ]]; then
-  # Retrieve and prepare CodeClimate's test coverage reporter.
-  curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
-  chmod +x ./cc-test-reporter
-  ./cc-test-reporter before-build
-fi
-
-# Run all the tests with the race detector and generate coverage.
-go test -v -race -coverprofile c.out ./...
-
-if [[ -n "${REPORT_COVERAGE+cover}" && "${TRAVIS_SECURE_ENV_VARS}" == "true" ]]; then
-  # Upload test coverage to CodeClimate.
-  ./cc-test-reporter after-build
-fi

contrib/outmessage-discordemoji.tengo

@@ -0,0 +1,10 @@
+text := import("text")
+
+// if we're not sending to a discord bridge,
+// then convert custom emoji tags into url's
+if (inProtocol == "discord" && outProtocol != "discord") {
+    rePNG := text.re_compile(`<:.*?:([0-9]+)>`)
+    msgText=rePNG.replace(msgText,"https://cdn.discordapp.com/emojis/$1.png")
+    reGIF := text.re_compile(`<a:.*?:([0-9]+)>`)
+    msgText=reGIF.replace(msgText,"https://cdn.discordapp.com/emojis/$1.gif")
+}

contrib/outmessage-irccolornick.tengo

@@ -0,0 +1,14 @@
+// See https://github.com/42wim/matterbridge/issues/881
+// Generates a colored nick for each msgUsername, with example to filter specific codes 
+
+text := import("text")
+fmt := import("fmt")
+if outProtocol == "irc" {
+    // generate a color for a nick, make sure it isn't 0 or 15
+    colorCode := len(msgUsername)+bytes(msgUsername)[0]%14 + 2
+    // example if we want to use colorCode 3 when we have calculated colorcode 14
+    if colorCode == 14 {
+        colorCode = 3
+    }
+    msgUsername=fmt.sprintf("\x03%02d%s\x0F", colorCode, msgUsername)
+}

contrib/outmessage-irccolors.tengo

@@ -0,0 +1,7 @@
+// See https://github.com/42wim/matterbridge/issues/798
+
+// if we're not sending to an irc bridge we strip the IRC colors
+if outProtocol != "irc" {
+    re := text.re_compile(`\x03(?:\d{1,2}(?:,\d{1,2})?)?|[[:cntrl:]]`)
+    msgText=re.replace(msgText,"")
+}

contrib/remotenickformat-zerowidth.tengo

@@ -0,0 +1,16 @@
+/*
+This script will return the nick except with multi-character usernames
+containing a zero-width space between the first and second character letter.
+
+Single character usernames will be left untouched.
+
+This is useful to prevent remote users from nickalerting
+IRC users of the same name when the remote user speaks.
+
+This result can be used in {TENGO} in RemoteNickFormat.
+*/
+
+result = nick
+if len(nick) > 1 {
+    result = string(nick[0]) + "​" + nick[1:]
+}

contrib/remotenickformat.tengo

@@ -0,0 +1,9 @@
+/*
+This script will return the current time in kitchen format if the protocol (of the remote bridge) isn't irc
+See https://github.com/d5/tengo/blob/master/docs/stdlib-times.md
+This result can be used in {TENGO} in RemoteNickFormat
+*/
+times := import("times")
+if protocol != "irc" {
+   result=times.time_format(times.now(),times.format_kitchen)
+}

gateway/bridgemap/api.go

@@ -0,0 +1,11 @@
+// +build !noapi
+
+package bridgemap
+
+import (
+	"github.com/42wim/matterbridge/bridge/api"
+)
+
+func init() {
+	FullMap["api"] = api.New
+}

gateway/bridgemap/bdiscord.go

@@ -0,0 +1,12 @@
+// +build !nodiscord
+
+package bridgemap
+
+import (
+	bdiscord "github.com/42wim/matterbridge/bridge/discord"
+)
+
+func init() {
+	FullMap["discord"] = bdiscord.New
+	UserTypingSupport["discord"] = struct{}{}
+}

gateway/bridgemap/bgitter.go

@@ -0,0 +1,11 @@
+// +build !nogitter
+
+package bridgemap
+
+import (
+	bgitter "github.com/42wim/matterbridge/bridge/gitter"
+)
+
+func init() {
+	FullMap["gitter"] = bgitter.New
+}

gateway/bridgemap/birc.go

@@ -0,0 +1,11 @@
+// +build !noirc
+
+package bridgemap
+
+import (
+	birc "github.com/42wim/matterbridge/bridge/irc"
+)
+
+func init() {
+	FullMap["irc"] = birc.New
+}

gateway/bridgemap/bkeybase.go

@@ -0,0 +1,11 @@
+// +build !nokeybase
+
+package bridgemap
+
+import (
+	bkeybase "github.com/42wim/matterbridge/bridge/keybase"
+)
+
+func init() {
+	FullMap["keybase"] = bkeybase.New
+}

gateway/bridgemap/bmatrix.go

@@ -0,0 +1,11 @@
+// +build !nomatrix
+
+package bridgemap
+
+import (
+	bmatrix "github.com/42wim/matterbridge/bridge/matrix"
+)
+
+func init() {
+	FullMap["matrix"] = bmatrix.New
+}

gateway/bridgemap/bmattermost.go

@@ -0,0 +1,11 @@
+// +build !nomattermost
+
+package bridgemap
+
+import (
+	bmattermost "github.com/42wim/matterbridge/bridge/mattermost"
+)
+
+func init() {
+	FullMap["mattermost"] = bmattermost.New
+}

gateway/bridgemap/bmsteams.go

@@ -0,0 +1,11 @@
+// +build !nomsteams
+
+package bridgemap
+
+import (
+	bmsteams "github.com/42wim/matterbridge/bridge/msteams"
+)
+
+func init() {
+	FullMap["msteams"] = bmsteams.New
+}

gateway/bridgemap/bnctalk.go

@@ -0,0 +1,11 @@
+// +build !nonctalk
+
+package bridgemap
+
+import (
+	btalk "github.com/42wim/matterbridge/bridge/nctalk"
+)
+
+func init() {
+	FullMap["nctalk"] = btalk.New
+}

gateway/bridgemap/bridgemap.go

@@ -2,36 +2,9 @@ package bridgemap
 
 import (
 	"github.com/42wim/matterbridge/bridge"
-	"github.com/42wim/matterbridge/bridge/api"
-	"github.com/42wim/matterbridge/bridge/discord"
-	"github.com/42wim/matterbridge/bridge/gitter"
-	"github.com/42wim/matterbridge/bridge/irc"
-	"github.com/42wim/matterbridge/bridge/matrix"
-	"github.com/42wim/matterbridge/bridge/mattermost"
-	"github.com/42wim/matterbridge/bridge/rocketchat"
-	"github.com/42wim/matterbridge/bridge/slack"
-	"github.com/42wim/matterbridge/bridge/sshchat"
-	"github.com/42wim/matterbridge/bridge/steam"
-	"github.com/42wim/matterbridge/bridge/telegram"
-	"github.com/42wim/matterbridge/bridge/whatsapp"
-	"github.com/42wim/matterbridge/bridge/xmpp"
-	"github.com/42wim/matterbridge/bridge/zulip"
 )
 
-var FullMap = map[string]bridge.Factory{
-	"api":          api.New,
-	"discord":      bdiscord.New,
-	"gitter":       bgitter.New,
-	"irc":          birc.New,
-	"mattermost":   bmattermost.New,
-	"matrix":       bmatrix.New,
-	"rocketchat":   brocketchat.New,
-	"slack-legacy": bslack.NewLegacy,
-	"slack":        bslack.New,
-	"sshchat":      bsshchat.New,
-	"steam":        bsteam.New,
-	"telegram":     btelegram.New,
-	"whatsapp":     bwhatsapp.New,
-	"xmpp":         bxmpp.New,
-	"zulip":        bzulip.New,
-}
+var (
+	FullMap           = map[string]bridge.Factory{}
+	UserTypingSupport = map[string]struct{}{}
+)

gateway/bridgemap/brocketchat.go

@@ -0,0 +1,11 @@
+// +build !norocketchat
+
+package bridgemap
+
+import (
+	brocketchat "github.com/42wim/matterbridge/bridge/rocketchat"
+)
+
+func init() {
+	FullMap["rocketchat"] = brocketchat.New
+}

gateway/bridgemap/bslack.go

@@ -0,0 +1,13 @@
+// +build !noslack
+
+package bridgemap
+
+import (
+	bslack "github.com/42wim/matterbridge/bridge/slack"
+)
+
+func init() {
+	FullMap["slack-legacy"] = bslack.NewLegacy
+	FullMap["slack"] = bslack.New
+	UserTypingSupport["slack"] = struct{}{}
+}

gateway/bridgemap/bsshchat.go

@@ -0,0 +1,11 @@
+// +build !nosshchat
+
+package bridgemap
+
+import (
+	bsshchat "github.com/42wim/matterbridge/bridge/sshchat"
+)
+
+func init() {
+	FullMap["sshchat"] = bsshchat.New
+}

gateway/bridgemap/bsteam.go

@@ -0,0 +1,11 @@
+// +build !nosteam
+
+package bridgemap
+
+import (
+	bsteam "github.com/42wim/matterbridge/bridge/steam"
+)
+
+func init() {
+	FullMap["steam"] = bsteam.New
+}

gateway/bridgemap/btelegram.go

@@ -0,0 +1,11 @@
+// +build !notelegram
+
+package bridgemap
+
+import (
+	btelegram "github.com/42wim/matterbridge/bridge/telegram"
+)
+
+func init() {
+	FullMap["telegram"] = btelegram.New
+}

gateway/bridgemap/bwhatsapp.go

@@ -0,0 +1,11 @@
+// +build !nowhatsapp
+
+package bridgemap
+
+import (
+	bwhatsapp "github.com/42wim/matterbridge/bridge/whatsapp"
+)
+
+func init() {
+	FullMap["whatsapp"] = bwhatsapp.New
+}

gateway/bridgemap/bxmpp.go

@@ -0,0 +1,11 @@
+// +build !noxmpp
+
+package bridgemap
+
+import (
+	bxmpp "github.com/42wim/matterbridge/bridge/xmpp"
+)
+
+func init() {
+	FullMap["xmpp"] = bxmpp.New
+}

gateway/bridgemap/bzulip.go

@@ -0,0 +1,11 @@
+// +build !nozulip
+
+package bridgemap
+
+import (
+	bzulip "github.com/42wim/matterbridge/bridge/zulip"
+)
+
+func init() {
+	FullMap["zulip"] = bzulip.New
+}

gateway/gateway.go

@@ -9,10 +9,11 @@ import (
 
 	"github.com/42wim/matterbridge/bridge"
 	"github.com/42wim/matterbridge/bridge/config"
-	"github.com/d5/tengo/script"
-	"github.com/d5/tengo/stdlib"
+	"github.com/42wim/matterbridge/internal"
+	"github.com/d5/tengo/v2"
+	"github.com/d5/tengo/v2/stdlib"
 	lru "github.com/hashicorp/golang-lru"
-	"github.com/peterhellberg/emojilib"
+	"github.com/matterbridge/emoji"
 	"github.com/sirupsen/logrus"
 )
 
@@ -84,6 +85,7 @@ func (gw *Gateway) FindCanonicalMsgID(protocol string, mID string) string {
 func (gw *Gateway) AddBridge(cfg *config.Bridge) error {
 	br := gw.Router.getBridge(cfg.Account)
 	if br == nil {
+		gw.checkConfig(cfg)
 		br = bridge.New(cfg)
 		br.Config = gw.Router.Config
 		br.General = &gw.BridgeValues().General
@@ -103,6 +105,19 @@ func (gw *Gateway) AddBridge(cfg *config.Bridge) error {
 	return nil
 }
 
+func (gw *Gateway) checkConfig(cfg *config.Bridge) {
+	match := false
+	for _, key := range gw.Router.Config.Viper().AllKeys() {
+		if strings.HasPrefix(key, strings.ToLower(cfg.Account)) {
+			match = true
+			break
+		}
+	}
+	if !match {
+		gw.logger.Fatalf("Account %s defined in gateway %s but no configuration found, exiting.", cfg.Account, gw.Name)
+	}
+}
+
 // AddConfig associates a new configuration with the gateway object.
 func (gw *Gateway) AddConfig(cfg *config.Gateway) error {
 	gw.Name = cfg.Name
@@ -291,8 +306,6 @@ func (gw *Gateway) ignoreMessage(msg *config.Message) bool {
 }
 
 func (gw *Gateway) modifyUsername(msg *config.Message, dest *bridge.Bridge) string {
-	br := gw.Bridges[msg.Account]
-	msg.Protocol = br.Protocol
 	if dest.GetBool("StripNick") {
 		re := regexp.MustCompile("[^a-zA-Z0-9]+")
 		msg.Username = re.ReplaceAllString(msg.Username, "")
@@ -300,6 +313,7 @@ func (gw *Gateway) modifyUsername(msg *config.Message, dest *bridge.Bridge) stri
 	nick := dest.GetString("RemoteNickFormat")
 
 	// loop to replace nicks
+	br := gw.Bridges[msg.Account]
 	for _, outer := range br.GetStringSlice2D("ReplaceNicks") {
 		search := outer[0]
 		replace := outer[1]
@@ -331,6 +345,11 @@ func (gw *Gateway) modifyUsername(msg *config.Message, dest *bridge.Bridge) stri
 	nick = strings.Replace(nick, "{LABEL}", br.GetString("Label"), -1)
 	nick = strings.Replace(nick, "{NICK}", msg.Username, -1)
 	nick = strings.Replace(nick, "{CHANNEL}", msg.Channel, -1)
+	tengoNick, err := gw.modifyUsernameTengo(msg, br)
+	if err != nil {
+		gw.logger.Errorf("modifyUsernameTengo error: %s", err)
+	}
+	nick = strings.Replace(nick, "{TENGO}", tengoNick, -1) //nolint:gocritic
 	return nick
 }
 
@@ -347,9 +366,12 @@ func (gw *Gateway) modifyMessage(msg *config.Message) {
 	if err := modifyMessageTengo(gw.BridgeValues().General.TengoModifyMessage, msg); err != nil {
 		gw.logger.Errorf("TengoModifyMessage failed: %s", err)
 	}
+	if err := modifyMessageTengo(gw.BridgeValues().Tengo.Message, msg); err != nil {
+		gw.logger.Errorf("Tengo.Message failed: %s", err)
+	}
 
 	// replace :emoji: to unicode
-	msg.Text = emojilib.Replace(msg.Text)
+	msg.Text = emoji.Sprint(msg.Text)
 
 	br := gw.Bridges[msg.Account]
 	// loop to replace messages
@@ -421,6 +443,11 @@ func (gw *Gateway) SendMessage(
 		msg.ParentID = "msg-parent-not-found"
 	}
 
+	err := gw.modifySendMessageTengo(rmsg, &msg, dest)
+	if err != nil {
+		gw.logger.Errorf("modifySendMessageTengo: %s", err)
+	}
+
 	// if we are using mattermost plugin account, send messages to MattermostPlugin channel
 	// that can be picked up by the mattermost matterbridge plugin
 	if dest.Account == "mattermost.plugin" {
@@ -486,7 +513,7 @@ func modifyMessageTengo(filename string, msg *config.Message) error {
 	if err != nil {
 		return err
 	}
-	s := script.New(res)
+	s := tengo.NewScript(res)
 	s.SetImports(stdlib.GetModuleMap(stdlib.AllModuleNames()...))
 	_ = s.Add("msgText", msg.Text)
 	_ = s.Add("msgUsername", msg.Username)
@@ -503,3 +530,77 @@ func modifyMessageTengo(filename string, msg *config.Message) error {
 	msg.Username = c.Get("msgUsername").String()
 	return nil
 }
+
+func (gw *Gateway) modifyUsernameTengo(msg *config.Message, br *bridge.Bridge) (string, error) {
+	filename := gw.BridgeValues().Tengo.RemoteNickFormat
+	if filename == "" {
+		return "", nil
+	}
+	res, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return "", err
+	}
+	s := tengo.NewScript(res)
+	s.SetImports(stdlib.GetModuleMap(stdlib.AllModuleNames()...))
+	_ = s.Add("result", "")
+	_ = s.Add("msgText", msg.Text)
+	_ = s.Add("msgUsername", msg.Username)
+	_ = s.Add("nick", msg.Username)
+	_ = s.Add("msgAccount", msg.Account)
+	_ = s.Add("msgChannel", msg.Channel)
+	_ = s.Add("channel", msg.Channel)
+	_ = s.Add("msgProtocol", msg.Protocol)
+	_ = s.Add("remoteAccount", br.Account)
+	_ = s.Add("protocol", br.Protocol)
+	_ = s.Add("bridge", br.Name)
+	_ = s.Add("gateway", gw.Name)
+	c, err := s.Compile()
+	if err != nil {
+		return "", err
+	}
+	if err := c.Run(); err != nil {
+		return "", err
+	}
+	return c.Get("result").String(), nil
+}
+
+func (gw *Gateway) modifySendMessageTengo(origmsg *config.Message, msg *config.Message, br *bridge.Bridge) error {
+	filename := gw.BridgeValues().Tengo.OutMessage
+	var res []byte
+	var err error
+	if filename == "" {
+		res, err = internal.Asset("tengo/outmessage.tengo")
+		if err != nil {
+			return err
+		}
+	} else {
+		res, err = ioutil.ReadFile(filename)
+		if err != nil {
+			return err
+		}
+	}
+	s := tengo.NewScript(res)
+	s.SetImports(stdlib.GetModuleMap(stdlib.AllModuleNames()...))
+	_ = s.Add("inAccount", origmsg.Account)
+	_ = s.Add("inProtocol", origmsg.Protocol)
+	_ = s.Add("inChannel", origmsg.Channel)
+	_ = s.Add("inGateway", origmsg.Gateway)
+	_ = s.Add("inEvent", origmsg.Event)
+	_ = s.Add("outAccount", br.Account)
+	_ = s.Add("outProtocol", br.Protocol)
+	_ = s.Add("outChannel", msg.Channel)
+	_ = s.Add("outGateway", gw.Name)
+	_ = s.Add("outEvent", msg.Event)
+	_ = s.Add("msgText", msg.Text)
+	_ = s.Add("msgUsername", msg.Username)
+	c, err := s.Compile()
+	if err != nil {
+		return err
+	}
+	if err := c.Run(); err != nil {
+		return err
+	}
+	msg.Text = c.Get("msgText").String()
+	msg.Username = c.Get("msgUsername").String()
+	return nil
+}

gateway/gateway_test.go

@@ -15,10 +15,15 @@ import (
 
 var testconfig = []byte(`
 [irc.freenode]
+server=""
 [mattermost.test]
+server=""
 [gitter.42wim]
+server=""
 [discord.test]
+server=""
 [slack.test]
+server=""
 
 [[gateway]]
     name = "bridge1"
@@ -44,10 +49,15 @@ var testconfig = []byte(`
 
 var testconfig2 = []byte(`
 [irc.freenode]
+server=""
 [mattermost.test]
+server=""
 [gitter.42wim]
+server=""
 [discord.test]
+server=""
 [slack.test]
+server=""
 
 [[gateway]]
     name = "bridge1"
@@ -87,8 +97,11 @@ var testconfig2 = []byte(`
 
 var testconfig3 = []byte(`
 [irc.zzz]
+server=""
 [telegram.zzz]
+server=""
 [slack.zzz]
+server=""
 [[gateway]]
 name="bridge"
 enable=true
@@ -176,7 +189,6 @@ func TestNewRouter(t *testing.T) {
 	assert.Equal(t, 1, len(r.Gateways))
 	assert.Equal(t, 4, len(r.Gateways["bridge1"].Bridges))
 	assert.Equal(t, 4, len(r.Gateways["bridge1"].Channels))
-
 	r = maketestRouter(testconfig2)
 	assert.Equal(t, 2, len(r.Gateways))
 	assert.Equal(t, 4, len(r.Gateways["bridge1"].Bridges))

gateway/handlers.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/42wim/matterbridge/bridge"
 	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/42wim/matterbridge/gateway/bridgemap"
 )
 
 // handleEventFailure handles failures and reconnects bridges.
@@ -168,7 +169,7 @@ func (gw *Gateway) ignoreEvent(event string, dest *bridge.Bridge) bool {
 	switch event {
 	case config.EventAvatarDownload:
 		// Avatar downloads are only relevant for telegram and mattermost for now
-		if dest.Protocol != "mattermost" && dest.Protocol != "telegram" {
+		if dest.Protocol != "mattermost" && dest.Protocol != "telegram" && dest.Protocol != "xmpp" {
 			return true
 		}
 	case config.EventJoinLeave:
@@ -178,7 +179,7 @@ func (gw *Gateway) ignoreEvent(event string, dest *bridge.Bridge) bool {
 		}
 	case config.EventTopicChange:
 		// only relay topic change when used in some way on other side
-		if dest.GetBool("ShowTopicChange") && dest.GetBool("SyncTopic") {
+		if !dest.GetBool("ShowTopicChange") && !dest.GetBool("SyncTopic") {
 			return true
 		}
 	}
@@ -190,6 +191,14 @@ func (gw *Gateway) ignoreEvent(event string, dest *bridge.Bridge) bool {
 func (gw *Gateway) handleMessage(rmsg *config.Message, dest *bridge.Bridge) []*BrMsgID {
 	var brMsgIDs []*BrMsgID
 
+	// Not all bridges support "user is typing" indications so skip the message
+	// if the targeted bridge does not support it.
+	if rmsg.Event == config.EventUserTyping {
+		if _, ok := bridgemap.UserTypingSupport[dest.Protocol]; !ok {
+			return nil
+		}
+	}
+
 	// if we have an attached file, or other info
 	if rmsg.Extra != nil && len(rmsg.Extra[config.EventFileFailureSize]) != 0 && rmsg.Text == "" {
 		return brMsgIDs

gateway/router.go

@@ -59,8 +59,14 @@ func NewRouter(rootLogger *logrus.Logger, cfg config.Config, bridgeMap map[strin
 // between them.
 func (r *Router) Start() error {
 	m := make(map[string]*bridge.Bridge)
+	if len(r.Gateways) == 0 {
+		return fmt.Errorf("no [[gateway]] configured. See https://github.com/42wim/matterbridge/wiki/How-to-create-your-config for more info")
+	}
 	for _, gw := range r.Gateways {
 		r.logger.Infof("Parsing gateway %s", gw.Name)
+		if len(gw.Bridges) == 0 {
+			return fmt.Errorf("no bridges configured for gateway %s. See https://github.com/42wim/matterbridge/wiki/How-to-create-your-config for more info", gw.Name)
+		}
 		for _, br := range gw.Bridges {
 			m[br.Account] = br
 		}
@@ -125,7 +131,11 @@ func (r *Router) handleReceive() {
 		r.handleEventGetChannelMembers(&msg)
 		r.handleEventFailure(&msg)
 		r.handleEventRejoinChannels(&msg)
-		idx := 0
+
+		// Set message protocol based on the account it came from
+		msg.Protocol = r.getBridge(msg.Account).Protocol
+
+		filesHandled := false
 		for _, gw := range r.Gateways {
 			// record all the message ID's of the different bridges
 			var msgIDs []*BrMsgID
@@ -134,17 +144,26 @@ func (r *Router) handleReceive() {
 			}
 			msg.Timestamp = time.Now()
 			gw.modifyMessage(&msg)
-			if idx == 0 {
+			if !filesHandled {
 				gw.handleFiles(&msg)
+				filesHandled = true
 			}
 			for _, br := range gw.Bridges {
 				msgIDs = append(msgIDs, gw.handleMessage(&msg, br)...)
 			}
-			// only add the message ID if it doesn't already exists
-			if _, ok := gw.Messages.Get(msg.Protocol + " " + msg.ID); !ok && msg.ID != "" {
-				gw.Messages.Add(msg.Protocol+" "+msg.ID, msgIDs)
+
+			if msg.ID != "" {
+				_, exists := gw.Messages.Get(msg.Protocol + " " + msg.ID)
+
+				// Only add the message ID if it doesn't already exist
+				//
+				// For some bridges we always add/update the message ID.
+				// This is necessary as msgIDs will change if a bridge returns
+				// a different ID in response to edits.
+				if !exists || msg.Protocol == "discord" {
+					gw.Messages.Add(msg.Protocol+" "+msg.ID, msgIDs)
+				}
 			}
-			idx++
 		}
 	}
 }

go.mod

@@ -3,72 +3,53 @@ module github.com/42wim/matterbridge
 require (
 	github.com/42wim/go-gitter v0.0.0-20170828205020-017310c2d557
 	github.com/Baozisoftware/qrcode-terminal-go v0.0.0-20170407111555-c0650d8dff0f
-	github.com/BurntSushi/toml v0.0.0-20170318202913-d94612f9fc14 // indirect
 	github.com/Jeffail/gabs v1.1.1 // indirect
-	github.com/Philipp15b/go-steam v1.0.1-0.20180818081528-681bd9573329
-	github.com/bwmarrin/discordgo v0.19.0
-	github.com/d5/tengo v1.20.0
-	github.com/dfordsoft/golib v0.0.0-20180902042739-76ee6ab99bec
-	github.com/fsnotify/fsnotify v1.4.7
-	github.com/go-telegram-bot-api/telegram-bot-api v4.6.5-0.20181225215658-ec221ba9ea45+incompatible
-	github.com/google/gops v0.3.5
+	github.com/Philipp15b/go-steam v1.0.1-0.20190816133340-b04c5a83c1c0
+	github.com/Rhymen/go-whatsapp v0.1.1-0.20200421062035-31e8111ac334
+	github.com/d5/tengo/v2 v2.6.0
+	github.com/davecgh/go-spew v1.1.1
+	github.com/fsnotify/fsnotify v1.4.9
+	github.com/go-telegram-bot-api/telegram-bot-api v1.0.1-0.20200524105306-7434b0456e81
+	github.com/gomarkdown/markdown v0.0.0-20200609195525-3f9352745725
+	github.com/google/gops v0.3.10
 	github.com/gopackage/ddp v0.0.0-20170117053602-652027933df4 // indirect
-	github.com/gopherjs/gopherjs v0.0.0-20180628210949-0892b62f0d9f // indirect
-	github.com/gorilla/schema v1.0.2
-	github.com/gorilla/websocket v1.4.0
-	github.com/hashicorp/golang-lru v0.5.0
-	github.com/hpcloud/tail v1.0.0 // indirect
-	github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7
-	github.com/jtolds/gls v4.2.1+incompatible // indirect
-	github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1 // indirect
-	github.com/kr/pretty v0.1.0 // indirect
-	github.com/labstack/echo/v4 v4.0.0
-	github.com/lrstanley/girc v0.0.0-20190210212025-51b8e096d398
-	github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5 // indirect
-	github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6 // indirect
-	github.com/matterbridge/Rocket.Chat.Go.SDK v0.0.0-20190210153444-cc9d05784d5d
-	github.com/matterbridge/go-whatsapp v0.0.1-0.20190301204034-f2f1b29d441b
-	github.com/matterbridge/go-xmpp v0.0.0-20180529212104-cd19799fba91
-	github.com/matterbridge/gomatrix v0.0.0-20190102230110-6f9631ca6dea
-	github.com/matterbridge/gozulipbot v0.0.0-20190212232658-7aa251978a18
-	github.com/matterbridge/logrus-prefixed-formatter v0.0.0-20180806162718-01618749af61
-	github.com/matterbridge/mautrix-whatsapp v0.0.0-20190301210046-3539cf52ed6e
-	github.com/mattermost/mattermost-server v5.5.0+incompatible
+	github.com/gorilla/schema v1.1.0
+	github.com/gorilla/websocket v1.4.2
+	github.com/hashicorp/golang-lru v0.5.4
+	github.com/jpillora/backoff v1.0.0
+	github.com/keybase/go-keybase-chat-bot v0.0.0-20200505163032-5cacf52379da
+	github.com/labstack/echo/v4 v4.1.16
+	github.com/lrstanley/girc v0.0.0-20190801035559-4fc93959e1a7
+	github.com/matterbridge/Rocket.Chat.Go.SDK v0.0.0-20200411204219-d5c18ce75048
+	github.com/matterbridge/discordgo v0.21.2-0.20200718144317-01fe5db6c78d
+	github.com/matterbridge/emoji v2.1.1-0.20191117213217-af507f6b02db+incompatible
+	github.com/matterbridge/go-xmpp v0.0.0-20200418225040-c8a3a57b4050
+	github.com/matterbridge/gomatrix v0.0.0-20200209224845-c2104d7936a6
+	github.com/matterbridge/gozulipbot v0.0.0-20200820220548-be5824faa913
+	github.com/matterbridge/logrus-prefixed-formatter v0.5.3-0.20200523233437-d971309a77ba
+	github.com/mattermost/mattermost-server/v5 v5.25.2
+	github.com/mattn/godown v0.0.0-20200217152941-afc959f6a561
 	github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect
+	github.com/missdeer/golib v1.0.3
 	github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474 // indirect
 	github.com/mrexodia/wray v0.0.0-20160318003008-78a2c1f284ff // indirect
 	github.com/nelsonken/gomf v0.0.0-20180504123937-a9dd2f9deae9
-	github.com/nicksnyder/go-i18n v1.4.0 // indirect
-	github.com/nlopes/slack v0.5.0
-	github.com/onsi/ginkgo v1.6.0 // indirect
-	github.com/onsi/gomega v1.4.1 // indirect
-	github.com/paulrosania/go-charset v0.0.0-20151028000031-621bb39fcc83
-	github.com/pborman/uuid v0.0.0-20160216163710-c55201b03606 // indirect
-	github.com/peterhellberg/emojilib v0.0.0-20190124112554-c18758d55320
-	github.com/pkg/errors v0.8.0 // indirect
+	github.com/paulrosania/go-charset v0.0.0-20190326053356-55c9d7a5834c
 	github.com/rs/xid v1.2.1
 	github.com/russross/blackfriday v1.5.2
 	github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca
-	github.com/shazow/ssh-chat v0.0.0-20190125184227-81d7e1686296
-	github.com/sirupsen/logrus v1.3.0
-	github.com/smartystreets/assertions v0.0.0-20180803164922-886ec427f6b9 // indirect
-	github.com/smartystreets/goconvey v0.0.0-20180222194500-ef6db91d284a // indirect
-	github.com/spf13/viper v1.3.1
-	github.com/stretchr/testify v1.3.0
-	github.com/technoweenie/multipartstreamer v1.0.1 // indirect
+	github.com/shazow/ssh-chat v1.8.3-0.20200308224626-80ddf1f43a98
+	github.com/sirupsen/logrus v1.6.0
+	github.com/slack-go/slack v0.6.5
+	github.com/spf13/viper v1.7.0
+	github.com/stretchr/testify v1.5.1
+	github.com/writeas/go-strip-markdown v2.0.1+incompatible
 	github.com/x-cray/logrus-prefixed-formatter v0.5.2 // indirect
-	github.com/zfjagann/golang-ring v0.0.0-20190106091943-a88bb6aef447
-	gitlab.com/golang-commonmark/html v0.0.0-20180917080848-cfaf75183c4a // indirect
-	gitlab.com/golang-commonmark/linkify v0.0.0-20180917065525-c22b7bdb1179 // indirect
-	gitlab.com/golang-commonmark/markdown v0.0.0-20181102083822-772775880e1f
-	gitlab.com/golang-commonmark/mdurl v0.0.0-20180912090424-e5bce34c34f2 // indirect
-	gitlab.com/golang-commonmark/puny v0.0.0-20180912090636-2cd490539afe // indirect
-	gitlab.com/opennota/wd v0.0.0-20180912061657-c5d65f63c638 // indirect
-	go.uber.org/atomic v1.3.2 // indirect
-	go.uber.org/multierr v1.1.0 // indirect
-	go.uber.org/zap v1.9.1 // indirect
-	golang.org/x/image v0.0.0-20190220214146-31aff87c08e9
-	gopkg.in/fsnotify.v1 v1.4.7 // indirect
-	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
-	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
+	github.com/yaegashi/msgraph.go v0.1.3
+	github.com/zfjagann/golang-ring v0.0.0-20190304061218-d34796e0a6c2
+	golang.org/x/image v0.0.0-20200618115811-c13761719519
+	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
+	gomod.garykim.dev/nc-talk v0.0.2
 )
+
+go 1.13

internal/bindata.go

@@ -0,0 +1,288 @@
+// Code generated by go-bindata. DO NOT EDIT.
+// sources:
+// tengo/outmessage.tengo
+
+package internal
+
+
+import (
+	"bytes"
+	"compress/gzip"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+func bindataRead(data []byte, name string) ([]byte, error) {
+	gz, err := gzip.NewReader(bytes.NewBuffer(data))
+	if err != nil {
+		return nil, fmt.Errorf("Read %q: %v", name, err)
+	}
+
+	var buf bytes.Buffer
+	_, err = io.Copy(&buf, gz)
+	clErr := gz.Close()
+
+	if err != nil {
+		return nil, fmt.Errorf("Read %q: %v", name, err)
+	}
+	if clErr != nil {
+		return nil, err
+	}
+
+	return buf.Bytes(), nil
+}
+
+
+type asset struct {
+	bytes []byte
+	info  fileInfoEx
+}
+
+type fileInfoEx interface {
+	os.FileInfo
+	MD5Checksum() string
+}
+
+type bindataFileInfo struct {
+	name        string
+	size        int64
+	mode        os.FileMode
+	modTime     time.Time
+	md5checksum string
+}
+
+func (fi bindataFileInfo) Name() string {
+	return fi.name
+}
+func (fi bindataFileInfo) Size() int64 {
+	return fi.size
+}
+func (fi bindataFileInfo) Mode() os.FileMode {
+	return fi.mode
+}
+func (fi bindataFileInfo) ModTime() time.Time {
+	return fi.modTime
+}
+func (fi bindataFileInfo) MD5Checksum() string {
+	return fi.md5checksum
+}
+func (fi bindataFileInfo) IsDir() bool {
+	return false
+}
+func (fi bindataFileInfo) Sys() interface{} {
+	return nil
+}
+
+var _bindataTengoOutmessagetengo = []byte(
+	"\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xc4\x91\x3d\x8f\xda\x40\x10\x86\xfb\xfd\x15\x13\x37\xb1\x2d\x07\xe7\xa3" +
+	"\xb3\x64\x59\x11\x45\x94\x2e\x8a\x92\x0a\xd0\xb1\xac\x07\x33\xd2\x7a\xc7\x1a\x8f\x31\x88\xe3\xbf\x9f\xcc\x01\x47" +
+	"\x7f\xc5\x75\xef\xae\x9e\x9d\x77\x1f\x4d\x9e\x9a\xbd\x15\xb2\x1b\x8f\x3d\xd8\xbd\x25\x3f\x45\x30\x82\xb6\xfe\xc2" +
+	"\xc1\x1f\x0b\x43\xe1\xa7\x73\x3c\x04\xcd\x80\xc2\x1f\x61\x65\xc7\x7e\xca\xf3\x9d\x0d\x01\x2f\xf1\x97\x55\x1c\xed" +
+	"\xd1\xf0\xa0\x77\x98\x07\x7d\xa3\x79\xd0\x3b\xce\x83\xde\xf8\xd7\x9e\x51\x48\xb1\x30\x6d\xdf\xfc\xc3\x83\x66\xd0" +
+	"\xf6\xcd\xff\x1e\x25\xd8\x16\x4d\x9a\x1b\xa3\x78\x50\x28\x4a\xa0\xb6\x63\xd1\x38\x9a\xce\x51\x62\x4c\x9e\x43\xaf" +
+	"\x42\x1d\x90\x38\x70\xec\x59\xfa\xe9\x8e\xb6\x30\xe2\x67\x41\x08\xac\xd0\x63\xa8\x29\x34\xa0\x0c\x36\x5c\xc0\x8d" +
+	"\x50\xdd\x20\x8c\x78\x7d\xac\x3b\x84\xdf\x7f\xe7\xb7\x01\xb4\x7d\xd0\x84\xb2\x84\x88\xc4\x45\x70\x32\x00\x00\x82" +
+	"\xd3\x3f\xa6\xfe\x99\xe0\x93\xe3\xb6\x23\x8f\xf1\x7a\x79\xf8\xfa\x23\xae\x8a\x65\x7d\xfa\x96\x7d\x3f\xc7\x55\x91" +
+	"\x5d\x63\x52\x25\xd5\xf3\x62\x51\xb8\xa0\xe2\x8b\xd5\x6a\x9d\x5c\xc6\x5c\x4d\x4b\xc1\x99\x60\xe7\xad\xc3\xf8\x26" +
+	"\x1f\x45\x89\x39\x9b\xf7\x6b\xe4\x29\x6d\x1f\x57\x00\x9f\x3e\xc6\x24\xcd\xcd\x4b\x00\x00\x00\xff\xff\x40\xb8\x54" +
+	"\xb8\x64\x02\x00\x00")
+
+func bindataTengoOutmessagetengoBytes() ([]byte, error) {
+	return bindataRead(
+		_bindataTengoOutmessagetengo,
+		"tengo/outmessage.tengo",
+	)
+}
+
+
+
+func bindataTengoOutmessagetengo() (*asset, error) {
+	bytes, err := bindataTengoOutmessagetengoBytes()
+	if err != nil {
+		return nil, err
+	}
+
+	info := bindataFileInfo{
+		name: "tengo/outmessage.tengo",
+		size: 612,
+		md5checksum: "",
+		mode: os.FileMode(420),
+		modTime: time.Unix(1555622139, 0),
+	}
+
+	a := &asset{bytes: bytes, info: info}
+
+	return a, nil
+}
+
+
+//
+// Asset loads and returns the asset for the given name.
+// It returns an error if the asset could not be found or
+// could not be loaded.
+//
+func Asset(name string) ([]byte, error) {
+	cannonicalName := strings.Replace(name, "\\", "/", -1)
+	if f, ok := _bindata[cannonicalName]; ok {
+		a, err := f()
+		if err != nil {
+			return nil, fmt.Errorf("Asset %s can't read by error: %v", name, err)
+		}
+		return a.bytes, nil
+	}
+	return nil, &os.PathError{Op: "open", Path: name, Err: os.ErrNotExist}
+}
+
+//
+// MustAsset is like Asset but panics when Asset would return an error.
+// It simplifies safe initialization of global variables.
+// nolint: deadcode
+//
+func MustAsset(name string) []byte {
+	a, err := Asset(name)
+	if err != nil {
+		panic("asset: Asset(" + name + "): " + err.Error())
+	}
+
+	return a
+}
+
+//
+// AssetInfo loads and returns the asset info for the given name.
+// It returns an error if the asset could not be found or could not be loaded.
+//
+func AssetInfo(name string) (os.FileInfo, error) {
+	cannonicalName := strings.Replace(name, "\\", "/", -1)
+	if f, ok := _bindata[cannonicalName]; ok {
+		a, err := f()
+		if err != nil {
+			return nil, fmt.Errorf("AssetInfo %s can't read by error: %v", name, err)
+		}
+		return a.info, nil
+	}
+	return nil, &os.PathError{Op: "open", Path: name, Err: os.ErrNotExist}
+}
+
+//
+// AssetNames returns the names of the assets.
+// nolint: deadcode
+//
+func AssetNames() []string {
+	names := make([]string, 0, len(_bindata))
+	for name := range _bindata {
+		names = append(names, name)
+	}
+	return names
+}
+
+//
+// _bindata is a table, holding each asset generator, mapped to its name.
+//
+var _bindata = map[string]func() (*asset, error){
+	"tengo/outmessage.tengo": bindataTengoOutmessagetengo,
+}
+
+//
+// AssetDir returns the file names below a certain
+// directory embedded in the file by go-bindata.
+// For example if you run go-bindata on data/... and data contains the
+// following hierarchy:
+//     data/
+//       foo.txt
+//       img/
+//         a.png
+//         b.png
+// then AssetDir("data") would return []string{"foo.txt", "img"}
+// AssetDir("data/img") would return []string{"a.png", "b.png"}
+// AssetDir("foo.txt") and AssetDir("notexist") would return an error
+// AssetDir("") will return []string{"data"}.
+//
+func AssetDir(name string) ([]string, error) {
+	node := _bintree
+	if len(name) != 0 {
+		cannonicalName := strings.Replace(name, "\\", "/", -1)
+		pathList := strings.Split(cannonicalName, "/")
+		for _, p := range pathList {
+			node = node.Children[p]
+			if node == nil {
+				return nil, &os.PathError{
+					Op: "open",
+					Path: name,
+					Err: os.ErrNotExist,
+				}
+			}
+		}
+	}
+	if node.Func != nil {
+		return nil, &os.PathError{
+			Op: "open",
+			Path: name,
+			Err: os.ErrNotExist,
+		}
+	}
+	rv := make([]string, 0, len(node.Children))
+	for childName := range node.Children {
+		rv = append(rv, childName)
+	}
+	return rv, nil
+}
+
+
+type bintree struct {
+	Func     func() (*asset, error)
+	Children map[string]*bintree
+}
+
+var _bintree = &bintree{Func: nil, Children: map[string]*bintree{
+	"tengo": {Func: nil, Children: map[string]*bintree{
+		"outmessage.tengo": {Func: bindataTengoOutmessagetengo, Children: map[string]*bintree{}},
+	}},
+}}
+
+// RestoreAsset restores an asset under the given directory
+func RestoreAsset(dir, name string) error {
+	data, err := Asset(name)
+	if err != nil {
+		return err
+	}
+	info, err := AssetInfo(name)
+	if err != nil {
+		return err
+	}
+	err = os.MkdirAll(_filePath(dir, filepath.Dir(name)), os.FileMode(0755))
+	if err != nil {
+		return err
+	}
+	err = ioutil.WriteFile(_filePath(dir, name), data, info.Mode())
+	if err != nil {
+		return err
+	}
+	return os.Chtimes(_filePath(dir, name), info.ModTime(), info.ModTime())
+}
+
+// RestoreAssets restores an asset under the given directory recursively
+func RestoreAssets(dir, name string) error {
+	children, err := AssetDir(name)
+	// File
+	if err != nil {
+		return RestoreAsset(dir, name)
+	}
+	// Dir
+	for _, child := range children {
+		err = RestoreAssets(dir, filepath.Join(name, child))
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func _filePath(dir, name string) string {
+	cannonicalName := strings.Replace(name, "\\", "/", -1)
+	return filepath.Join(append([]string{dir}, strings.Split(cannonicalName, "/")...)...)
+}

internal/tengo/outmessage.tengo

@@ -0,0 +1,25 @@
+/*
+variables available 
+read-only:
+inAccount, inProtocol, inChannel, inGateway
+outAccount, outProtocol, outChannel, outGateway
+
+read-write:
+msgText, msgUsername
+*/
+
+text := import("text")
+
+// start - strip irc colors 
+// if we're not sending to an irc bridge we strip the IRC colors
+if inProtocol == "irc" && outProtocol != "irc" {
+    re := text.re_compile(`\x03(?:\d{1,2}(?:,\d{1,2})?)?|[[:cntrl:]]`)
+    msgText=re.replace(msgText,"")
+}
+// end - strip irc colors
+
+// strip custom emoji
+if inProtocol == "discord" {
+    re := text.re_compile(`<a?(:.*?:)[0-9]+>`)
+    msgText=re.replace(msgText,"$1")
+}

matterbridge.go

@@ -4,6 +4,7 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"runtime"
 	"strings"
 
 	"github.com/42wim/matterbridge/bridge/config"
@@ -15,7 +16,7 @@ import (
 )
 
 var (
-	version = "1.14.2"
+	version = "1.18.1"
 	githash string
 
 	flagConfig  = flag.String("conf", "matterbridge.toml", "config file")
@@ -50,6 +51,15 @@ func main() {
 	cfg := config.NewConfig(rootLogger, *flagConfig)
 	cfg.BridgeValues().General.Debug = *flagDebug
 
+	// if logging to a file, ensure it is closed when the program terminates
+	// nolint:errcheck
+	defer func() {
+		if f, ok := rootLogger.Out.(*os.File); ok {
+			f.Sync()
+			f.Close()
+		}
+	}()
+
 	r, err := gateway.NewRouter(rootLogger, cfg, bridgemap.FullMap)
 	if err != nil {
 		logger.Fatalf("Starting gateway failed: %s", err)
@@ -67,17 +77,31 @@ func setupLogger() *logrus.Logger {
 		Formatter: &prefixed.TextFormatter{
 			PrefixPadding: 13,
 			DisableColors: true,
-			FullTimestamp: true,
 		},
 		Level: logrus.InfoLevel,
 	}
 	if *flagDebug || os.Getenv("DEBUG") == "1" {
+		logger.SetReportCaller(true)
 		logger.Formatter = &prefixed.TextFormatter{
-			PrefixPadding:   13,
-			DisableColors:   true,
-			FullTimestamp:   false,
-			ForceFormatting: true,
+			PrefixPadding: 13,
+			DisableColors: true,
+			FullTimestamp: false,
+
+			CallerFormatter: func(function, file string) string {
+				return fmt.Sprintf(" [%s:%s]", function, file)
+			},
+			CallerPrettyfier: func(f *runtime.Frame) (string, string) {
+				sp := strings.SplitAfter(f.File, "/matterbridge/")
+				filename := f.File
+				if len(sp) > 1 {
+					filename = sp[1]
+				}
+				s := strings.Split(f.Function, ".")
+				funcName := s[len(s)-1]
+				return funcName, fmt.Sprintf("%s:%d", filename, f.Line)
+			},
 		}
+
 		logger.Level = logrus.DebugLevel
 		logger.WithFields(logrus.Fields{"prefix": "main"}).Info("Enabling debug logging.")
 	}

matterclient/channels.go

@@ -4,7 +4,7 @@ import (
 	"errors"
 	"strings"
 
-	"github.com/mattermost/mattermost-server/model"
+	"github.com/mattermost/mattermost-server/v5/model"
 )
 
 // GetChannels returns all channels we're members off
@@ -36,6 +36,16 @@ func (m *MMClient) GetChannelHeader(channelId string) string { //nolint:golint
 	return ""
 }
 
+func getNormalisedName(channel *model.Channel) string {
+	if channel.Type == model.CHANNEL_GROUP {
+		// (deprecated in favor of ReplaceAll in go 1.12)
+		res := strings.Replace(channel.DisplayName, ", ", "-", -1) //nolint: gocritic
+		res = strings.Replace(res, " ", "_", -1)                   //nolint: gocritic
+		return res
+	}
+	return channel.Name
+}
+
 func (m *MMClient) GetChannelId(name string, teamId string) string { //nolint:golint
 	m.RLock()
 	defer m.RUnlock()
@@ -45,13 +55,7 @@ func (m *MMClient) GetChannelId(name string, teamId string) string { //nolint:go
 
 	for _, t := range m.OtherTeams {
 		for _, channel := range append(t.Channels, t.MoreChannels...) {
-			if channel.Type == model.CHANNEL_GROUP {
-				res := strings.Replace(channel.DisplayName, ", ", "-", -1)
-				res = strings.Replace(res, " ", "_", -1)
-				if res == name {
-					return channel.Id
-				}
-			} else if channel.Name == name {
+			if getNormalisedName(channel) == name {
 				return channel.Id
 			}
 		}
@@ -63,7 +67,7 @@ func (m *MMClient) getChannelIdTeam(name string, teamId string) string { //nolin
 	for _, t := range m.OtherTeams {
 		if t.Id == teamId {
 			for _, channel := range append(t.Channels, t.MoreChannels...) {
-				if channel.Name == name {
+				if getNormalisedName(channel) == name {
 					return channel.Id
 				}
 			}
@@ -81,12 +85,7 @@ func (m *MMClient) GetChannelName(channelId string) string { //nolint:golint
 		}
 		for _, channel := range append(t.Channels, t.MoreChannels...) {
 			if channel.Id == channelId {
-				if channel.Type == model.CHANNEL_GROUP {
-					res := strings.Replace(channel.DisplayName, ", ", "-", -1)
-					res = strings.Replace(res, " ", "_", -1)
-					return res
-				}
-				return channel.Name
+				return getNormalisedName(channel)
 			}
 		}
 	}
@@ -167,23 +166,42 @@ func (m *MMClient) JoinChannel(channelId string) error { //nolint:golint
 	return nil
 }
 
+func (m *MMClient) UpdateChannelsTeam(teamID string) error {
+	mmchannels, resp := m.Client.GetChannelsForTeamForUser(teamID, m.User.Id, false, "")
+	if resp.Error != nil {
+		return errors.New(resp.Error.DetailedError)
+	}
+	for idx, t := range m.OtherTeams {
+		if t.Id == teamID {
+			m.Lock()
+			m.OtherTeams[idx].Channels = mmchannels
+			m.Unlock()
+		}
+	}
+
+	mmchannels, resp = m.Client.GetPublicChannelsForTeam(teamID, 0, 5000, "")
+	if resp.Error != nil {
+		return errors.New(resp.Error.DetailedError)
+	}
+	for idx, t := range m.OtherTeams {
+		if t.Id == teamID {
+			m.Lock()
+			m.OtherTeams[idx].MoreChannels = mmchannels
+			m.Unlock()
+		}
+	}
+	return nil
+}
+
 func (m *MMClient) UpdateChannels() error {
-	mmchannels, resp := m.Client.GetChannelsForTeamForUser(m.Team.Id, m.User.Id, "")
-	if resp.Error != nil {
-		return errors.New(resp.Error.DetailedError)
+	if err := m.UpdateChannelsTeam(m.Team.Id); err != nil {
+		return err
 	}
-	m.Lock()
-	m.Team.Channels = mmchannels
-	m.Unlock()
-
-	mmchannels, resp = m.Client.GetPublicChannelsForTeam(m.Team.Id, 0, 5000, "")
-	if resp.Error != nil {
-		return errors.New(resp.Error.DetailedError)
+	for _, t := range m.OtherTeams {
+		if err := m.UpdateChannelsTeam(t.Id); err != nil {
+			return err
+		}
 	}
-
-	m.Lock()
-	m.Team.MoreChannels = mmchannels
-	m.Unlock()
 	return nil
 }
 

matterclient/helpers.go

@@ -13,7 +13,7 @@ import (
 
 	"github.com/gorilla/websocket"
 	"github.com/jpillora/backoff"
-	"github.com/mattermost/mattermost-server/model"
+	"github.com/mattermost/mattermost-server/v5/model"
 )
 
 func (m *MMClient) doLogin(firstConnection bool, b *backoff.Backoff) error {
@@ -132,18 +132,29 @@ func (m *MMClient) initUser() error {
 		return resp.Error
 	}
 	for _, team := range teams {
-		mmusers, resp := m.Client.GetUsersInTeam(team.Id, 0, 50000, "")
+		idx := 0
+		max := 200
+		usermap := make(map[string]*model.User)
+		mmusers, resp := m.Client.GetUsersInTeam(team.Id, idx, max, "")
 		if resp.Error != nil {
 			return errors.New(resp.Error.DetailedError)
 		}
-		usermap := make(map[string]*model.User)
-		for _, user := range mmusers {
-			usermap[user.Id] = user
+		for len(mmusers) > 0 {
+			for _, user := range mmusers {
+				usermap[user.Id] = user
+			}
+			mmusers, resp = m.Client.GetUsersInTeam(team.Id, idx, max, "")
+			if resp.Error != nil {
+				return errors.New(resp.Error.DetailedError)
+			}
+			idx++
+			time.Sleep(time.Millisecond * 200)
 		}
+		m.logger.Infof("found %d users in team %s", len(usermap), team.Name)
 
 		t := &Team{Team: team, Users: usermap, Id: team.Id}
 
-		mmchannels, resp := m.Client.GetChannelsForTeamForUser(team.Id, m.User.Id, "")
+		mmchannels, resp := m.Client.GetChannelsForTeamForUser(team.Id, m.User.Id, false, "")
 		if resp.Error != nil {
 			return resp.Error
 		}
@@ -175,15 +186,19 @@ func (m *MMClient) serverAlive(firstConnection bool, b *backoff.Backoff) error {
 		if resp.Error != nil {
 			return fmt.Errorf("%#v", resp.Error.Error())
 		}
-		if firstConnection && !supportedVersion(resp.ServerVersion) {
+		if firstConnection && !m.SkipVersionCheck && !supportedVersion(resp.ServerVersion) {
 			return fmt.Errorf("unsupported mattermost version: %s", resp.ServerVersion)
 		}
-		m.ServerVersion = resp.ServerVersion
-		if m.ServerVersion == "" {
-			m.logger.Debugf("Server not up yet, reconnecting in %s", d)
-			time.Sleep(d)
+		if !m.SkipVersionCheck {
+			m.ServerVersion = resp.ServerVersion
+			if m.ServerVersion == "" {
+				m.logger.Debugf("Server not up yet, reconnecting in %s", d)
+				time.Sleep(d)
+			} else {
+				m.logger.Infof("Found version %s", m.ServerVersion)
+				return nil
+			}
 		} else {
-			m.logger.Infof("Found version %s", m.ServerVersion)
 			return nil
 		}
 	}

matterclient/matterclient.go

@@ -11,19 +11,20 @@ import (
 	lru "github.com/hashicorp/golang-lru"
 	"github.com/jpillora/backoff"
 	prefixed "github.com/matterbridge/logrus-prefixed-formatter"
-	"github.com/mattermost/mattermost-server/model"
+	"github.com/mattermost/mattermost-server/v5/model"
 	"github.com/sirupsen/logrus"
 )
 
 type Credentials struct {
-	Login         string
-	Team          string
-	Pass          string
-	Token         string
-	CookieToken   bool
-	Server        string
-	NoTLS         bool
-	SkipTLSVerify bool
+	Login            string
+	Team             string
+	Pass             string
+	Token            string
+	CookieToken      bool
+	Server           string
+	NoTLS            bool
+	SkipTLSVerify    bool
+	SkipVersionCheck bool
 }
 
 type Message struct {
@@ -68,6 +69,7 @@ type MMClient struct {
 	logger     *logrus.Entry
 	rootLogger *logrus.Logger
 	lruCache   *lru.Cache
+	allevents  bool
 }
 
 // New will instantiate a new Matterclient with the specified login details without connecting.
@@ -118,6 +120,10 @@ func (m *MMClient) SetLogLevel(level string) {
 	}
 }
 
+func (m *MMClient) EnableAllEvents() {
+	m.allevents = true
+}
+
 // Login tries to connect the client with the loging details with which it was initialized.
 func (m *MMClient) Login() error {
 	// check if this is a first connect or a reconnection
@@ -219,8 +225,15 @@ func (m *MMClient) WsReceiver() {
 					continue
 				}
 			}
+			if m.allevents {
+				m.MessageChan <- msg
+				continue
+			}
 			switch msg.Raw.Event {
-			case model.WEBSOCKET_EVENT_USER_ADDED, model.WEBSOCKET_EVENT_USER_REMOVED:
+			case model.WEBSOCKET_EVENT_USER_ADDED,
+				model.WEBSOCKET_EVENT_USER_REMOVED,
+				model.WEBSOCKET_EVENT_CHANNEL_CREATED,
+				model.WEBSOCKET_EVENT_CHANNEL_DELETED:
 				m.MessageChan <- msg
 				continue
 			}

matterclient/messages.go

@@ -3,7 +3,7 @@ package matterclient
 import (
 	"strings"
 
-	"github.com/mattermost/mattermost-server/model"
+	"github.com/mattermost/mattermost-server/v5/model"
 )
 
 func (m *MMClient) parseActionPost(rmsg *Message) {
@@ -83,7 +83,7 @@ func (m *MMClient) DeleteMessage(postId string) error { //nolint:golint
 }
 
 func (m *MMClient) EditMessage(postId string, text string) (string, error) { //nolint:golint
-	post := &model.Post{Message: text}
+	post := &model.Post{Message: text, Id: postId}
 	res, resp := m.Client.UpdatePost(postId, post)
 	if resp.Error != nil {
 		return "", resp.Error

matterclient/users.go

@@ -2,8 +2,9 @@ package matterclient
 
 import (
 	"errors"
+	"time"
 
-	"github.com/mattermost/mattermost-server/model"
+	"github.com/mattermost/mattermost-server/v5/model"
 )
 
 func (m *MMClient) GetNickName(userId string) string { //nolint:golint
@@ -99,15 +100,25 @@ func (m *MMClient) GetUsers() map[string]*model.User {
 }
 
 func (m *MMClient) UpdateUsers() error {
-	mmusers, resp := m.Client.GetUsers(0, 50000, "")
+	idx := 0
+	max := 200
+	mmusers, resp := m.Client.GetUsers(idx, max, "")
 	if resp.Error != nil {
 		return errors.New(resp.Error.DetailedError)
 	}
-	m.Lock()
-	for _, user := range mmusers {
-		m.Users[user.Id] = user
+	for len(mmusers) > 0 {
+		m.Lock()
+		for _, user := range mmusers {
+			m.Users[user.Id] = user
+		}
+		m.Unlock()
+		mmusers, resp = m.Client.GetUsers(idx, max, "")
+		time.Sleep(time.Millisecond * 300)
+		if resp.Error != nil {
+			return errors.New(resp.Error.DetailedError)
+		}
+		idx++
 	}
-	m.Unlock()
 	return nil
 }
 

matterhook/matterhook.go

@@ -14,7 +14,7 @@ import (
 	"time"
 
 	"github.com/gorilla/schema"
-	"github.com/nlopes/slack"
+	"github.com/slack-go/slack"
 )
 
 // OMessage for mattermost incoming webhook. (send to mattermost)

vendor/github.com/Philipp15b/go-steam/README.md

@@ -45,20 +45,11 @@ Whether you want to develop your own Steam bot or directly work on go-steam itse
 
 ## Updating go-steam to a new SteamKit version
 
-To update go-steam to a new version of SteamKit, do the following:
+Go source code is generated with code in the `generator` directory.
+Look at `generator/README.md` for more information on how to use the generator.
 
-	go get github.com/golang/protobuf/protoc-gen-go/
-    git submodule init && git submodule update
-    cd generator
-    go run generator.go clean proto steamlang
-
-Make sure that `$GOPATH/bin` / `protoc-gen-go` is in your `$PATH`. You'll also need [`protoc`](https://developers.google.com/protocol-buffers/docs/downloads), the protocol buffer compiler. At the moment, we use Protocol Buffers 2.6.1 with `proco-gen-go`-[2402d76](https://github.com/golang/protobuf/tree/2402d76f3d41f928c7902a765dfc872356dd3aad).
-
-To compile the Steam Language files, you also need the [.NET Framework](https://www.microsoft.com/net/downloads)
-on Windows or [mono](http://www.go-mono.com/mono-downloads/download.html) on other operating systems.
-
-Apply the protocol changes where necessary.
+Then, after generating new Go source files, update `go-steam` as necessary.
 
 ## License
 
-Steam for Go is licensed under the New BSD License. More information can be found in LICENSE.txt.
+Steam for Go is licensed under the New BSD License. More information can be found in LICENSE.txt.

vendor/github.com/Philipp15b/go-steam/auth.go

@@ -94,9 +94,6 @@ func (a *Auth) HandlePacket(packet *Packet) {
 		a.handleUpdateMachineAuth(packet)
 	case EMsg_ClientAccountInfo:
 		a.handleAccountInfo(packet)
-	case EMsg_ClientWalletInfoUpdate:
-	case EMsg_ClientRequestWebAPIAuthenticateUserNonceResponse:
-	case EMsg_ClientMarketingMessageUpdate:
 	}
 }
 

vendor/github.com/Philipp15b/go-steam/client.go

@@ -133,11 +133,17 @@ func (c *Client) Connected() bool {
 // If you want to connect to a specific server, use `ConnectTo`.
 func (c *Client) Connect() *netutil.PortAddr {
 	var server *netutil.PortAddr
+
+	// try to initialize the directory cache
+	if !steamDirectoryCache.IsInitialized() {
+		_ = steamDirectoryCache.Initialize()
+	}
 	if steamDirectoryCache.IsInitialized() {
 		server = steamDirectoryCache.GetRandomCM()
 	} else {
 		server = GetRandomCM()
 	}
+
 	c.ConnectTo(server)
 	return server
 }

vendor/github.com/Philipp15b/go-steam/protocol/internal.go

@@ -1,6 +1,7 @@
 package protocol
 
 import (
+	"encoding/hex"
 	"io"
 	"math"
 	"strconv"
@@ -42,6 +43,7 @@ const EClientPersonaStateFlag_DefaultInfoRequest = EClientPersonaStateFlag_Playe
 
 const DefaultAvatar = "fef49e7fa7e1997310d705b2a6158ff8dc1cdfeb"
 
-func ValidAvatar(avatar string) bool {
-	return !(avatar == "0000000000000000000000000000000000000000" || len(avatar) != 40)
+func ValidAvatar(avatar []byte) bool {
+	str := hex.EncodeToString(avatar)
+	return !(str == "0000000000000000000000000000000000000000" || len(str) != 40)
 }

vendor/github.com/Philipp15b/go-steam/protocol/protobuf/app_ticket.pb.go

@@ -1,31 +1,60 @@
-// Code generated by protoc-gen-go.
+// Code generated by protoc-gen-go. DO NOT EDIT.
 // source: encrypted_app_ticket.proto
-// DO NOT EDIT!
 
 package protobuf
 
-import proto "github.com/golang/protobuf/proto"
-import fmt "fmt"
-import math "math"
+import (
+	fmt "fmt"
+	proto "github.com/golang/protobuf/proto"
+	math "math"
+)
 
 // Reference imports to suppress errors if they are not otherwise used.
 var _ = proto.Marshal
 var _ = fmt.Errorf
 var _ = math.Inf
 
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package protobuf is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package protobuf to be updated.
+const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
+
 type EncryptedAppTicket struct {
-	TicketVersionNo               *uint32 `protobuf:"varint,1,opt,name=ticket_version_no" json:"ticket_version_no,omitempty"`
-	CrcEncryptedticket            *uint32 `protobuf:"varint,2,opt,name=crc_encryptedticket" json:"crc_encryptedticket,omitempty"`
-	CbEncrypteduserdata           *uint32 `protobuf:"varint,3,opt,name=cb_encrypteduserdata" json:"cb_encrypteduserdata,omitempty"`
-	CbEncryptedAppownershipticket *uint32 `protobuf:"varint,4,opt,name=cb_encrypted_appownershipticket" json:"cb_encrypted_appownershipticket,omitempty"`
-	EncryptedTicket               []byte  `protobuf:"bytes,5,opt,name=encrypted_ticket" json:"encrypted_ticket,omitempty"`
-	XXX_unrecognized              []byte  `json:"-"`
+	TicketVersionNo               *uint32  `protobuf:"varint,1,opt,name=ticket_version_no" json:"ticket_version_no,omitempty"`
+	CrcEncryptedticket            *uint32  `protobuf:"varint,2,opt,name=crc_encryptedticket" json:"crc_encryptedticket,omitempty"`
+	CbEncrypteduserdata           *uint32  `protobuf:"varint,3,opt,name=cb_encrypteduserdata" json:"cb_encrypteduserdata,omitempty"`
+	CbEncryptedAppownershipticket *uint32  `protobuf:"varint,4,opt,name=cb_encrypted_appownershipticket" json:"cb_encrypted_appownershipticket,omitempty"`
+	EncryptedTicket               []byte   `protobuf:"bytes,5,opt,name=encrypted_ticket" json:"encrypted_ticket,omitempty"`
+	XXX_NoUnkeyedLiteral          struct{} `json:"-"`
+	XXX_unrecognized              []byte   `json:"-"`
+	XXX_sizecache                 int32    `json:"-"`
 }
 
-func (m *EncryptedAppTicket) Reset()                    { *m = EncryptedAppTicket{} }
-func (m *EncryptedAppTicket) String() string            { return proto.CompactTextString(m) }
-func (*EncryptedAppTicket) ProtoMessage()               {}
-func (*EncryptedAppTicket) Descriptor() ([]byte, []int) { return app_ticket_fileDescriptor0, []int{0} }
+func (m *EncryptedAppTicket) Reset()         { *m = EncryptedAppTicket{} }
+func (m *EncryptedAppTicket) String() string { return proto.CompactTextString(m) }
+func (*EncryptedAppTicket) ProtoMessage()    {}
+func (*EncryptedAppTicket) Descriptor() ([]byte, []int) {
+	return fileDescriptor_c6d69fd1cac4e8d5, []int{0}
+}
+
+func (m *EncryptedAppTicket) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_EncryptedAppTicket.Unmarshal(m, b)
+}
+func (m *EncryptedAppTicket) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_EncryptedAppTicket.Marshal(b, m, deterministic)
+}
+func (m *EncryptedAppTicket) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_EncryptedAppTicket.Merge(m, src)
+}
+func (m *EncryptedAppTicket) XXX_Size() int {
+	return xxx_messageInfo_EncryptedAppTicket.Size(m)
+}
+func (m *EncryptedAppTicket) XXX_DiscardUnknown() {
+	xxx_messageInfo_EncryptedAppTicket.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_EncryptedAppTicket proto.InternalMessageInfo
 
 func (m *EncryptedAppTicket) GetTicketVersionNo() uint32 {
 	if m != nil && m.TicketVersionNo != nil {
@@ -66,17 +95,19 @@ func init() {
 	proto.RegisterType((*EncryptedAppTicket)(nil), "EncryptedAppTicket")
 }
 
-var app_ticket_fileDescriptor0 = []byte{
-	// 162 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0x92, 0x4a, 0xcd, 0x4b, 0x2e,
+func init() { proto.RegisterFile("encrypted_app_ticket.proto", fileDescriptor_c6d69fd1cac4e8d5) }
+
+var fileDescriptor_c6d69fd1cac4e8d5 = []byte{
+	// 164 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x4a, 0xcd, 0x4b, 0x2e,
 	0xaa, 0x2c, 0x28, 0x49, 0x4d, 0x89, 0x4f, 0x2c, 0x28, 0x88, 0x2f, 0xc9, 0x4c, 0xce, 0x4e, 0x2d,
 	0xd1, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x57, 0x5a, 0xcb, 0xc8, 0x25, 0xe4, 0x0a, 0x93, 0x76, 0x2c,
 	0x28, 0x08, 0x01, 0x4b, 0x0a, 0x49, 0x72, 0x09, 0x42, 0x94, 0xc5, 0x97, 0xa5, 0x16, 0x15, 0x67,
 	0xe6, 0xe7, 0xc5, 0xe7, 0xe5, 0x4b, 0x30, 0x2a, 0x30, 0x6a, 0xf0, 0x0a, 0x49, 0x73, 0x09, 0x27,
 	0x17, 0x25, 0xc7, 0xc3, 0xcd, 0x84, 0xa8, 0x93, 0x60, 0x02, 0x4b, 0xca, 0x70, 0x89, 0x24, 0x27,
 	0x21, 0xe4, 0x4a, 0x8b, 0x53, 0x8b, 0x52, 0x12, 0x4b, 0x12, 0x25, 0x98, 0xc1, 0xb2, 0xea, 0x5c,
-	0xf2, 0xc8, 0xb2, 0x20, 0xd7, 0xe4, 0x97, 0xe7, 0x01, 0x2d, 0xc8, 0xc8, 0x2c, 0x80, 0x1a, 0xc3,
-	0x02, 0x56, 0x28, 0xc1, 0x25, 0x80, 0x50, 0x05, 0x95, 0x61, 0x05, 0xca, 0xf0, 0x38, 0xb1, 0x7a,
-	0x30, 0x36, 0x30, 0x32, 0x00, 0x02, 0x00, 0x00, 0xff, 0xff, 0x03, 0x8c, 0xdb, 0x92, 0xd3, 0x00,
-	0x00, 0x00,
+	0xf2, 0xc8, 0xb2, 0x20, 0xd7, 0xe4, 0x97, 0xe7, 0xa5, 0x16, 0x15, 0x67, 0x64, 0x16, 0x40, 0x8d,
+	0x61, 0x01, 0x2b, 0x94, 0xe0, 0x12, 0x40, 0xa8, 0x82, 0xca, 0xb0, 0x2a, 0x30, 0x6a, 0xf0, 0x38,
+	0xb1, 0x7a, 0x30, 0x36, 0x30, 0x32, 0x00, 0x02, 0x00, 0x00, 0xff, 0xff, 0x03, 0x8c, 0xdb, 0x92,
+	0xd3, 0x00, 0x00, 0x00,
 }