Release v1.22.2 (#1504)

* nctalk: add message deletion support

Signed-off-by: Gary Kim <gary@garykim.dev>

* nctalk: seperate out deletion and sending logic

Signed-off-by: Gary Kim <gary@garykim.dev>

* nctalk: update library to v0.2.0

Signed-off-by: Gary Kim <gary@garykim.dev>

* Rename functions to be clearer

Signed-off-by: Gary Kim <gary@garykim.dev>

* Update to go-nc-talk v0.2.1

Signed-off-by: Gary Kim <gary@garykim.dev>

* Update to go-nc-talk v0.2.2

Signed-off-by: Gary Kim <gary@garykim.dev>

* Make deletions easier to debug

Signed-off-by: Gary Kim <gary@garykim.dev>

.dockerignore

@@ -0,0 +1,2 @@
+Dockerfile
+tgs.Dockerfile

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,71 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+name: "CodeQL"
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [master]
+  schedule:
+    - cron: '0 16 * * 1'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        # Override automatic language detection by changing the below list
+        # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python']
+        language: ['go']
+        # Learn more...
+        # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+      with:
+        # We must fetch at least the immediate parents so that if this is
+        # a pull request then we can checkout the head.
+        fetch-depth: 2
+
+    # If this run was triggered by a pull request event, then checkout
+    # the head of the pull request instead of the merge commit.
+    - run: git checkout HEAD^2
+      if: ${{ github.event_name == 'pull_request' }}
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file. 
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/development.yml

@@ -0,0 +1,58 @@
+name: Development
+on: [push, pull_request]
+jobs:
+  lint:
+    name: golangci-lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 20
+      - name: Run golangci-lint
+        uses: golangci/golangci-lint-action@v2
+        with:
+          version: v1.29
+          args: "-v --new-from-rev HEAD~5"
+  test-build-upload:
+    strategy:
+      matrix:
+        go-version: [1.15.x, 1.16.x]
+        platform: [ubuntu-latest]
+    runs-on: ${{ matrix.platform }}
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+        stable: false
+    - name: Checkout code
+      uses: actions/checkout@v2
+      with:
+          fetch-depth: 0
+    - name: Test
+      run: go test ./... -mod=vendor
+    - name: Build
+      run: |
+        mkdir -p output/{win,lin,arm,mac}
+        VERSION=$(git describe --tags)
+        GOOS=linux GOARCH=amd64 go build -ldflags "-s -X main.githash=$(git log --pretty=format:'%h' -n 1)" -o output/lin/matterbridge-$VERSION-linux-amd64
+        GOOS=windows GOARCH=amd64 go build -ldflags "-s -X main.githash=$(git log --pretty=format:'%h' -n 1)" -o output/win/matterbridge-$VERSION-windows-amd64.exe
+        GOOS=darwin GOARCH=amd64 go build -ldflags "-s -X main.githash=$(git log --pretty=format:'%h' -n 1)" -o output/mac/matterbridge-$VERSION-darwin-amd64
+    - name: Upload linux 64-bit
+      if: startsWith(matrix.go-version,'1.16')
+      uses: actions/upload-artifact@v2
+      with:
+        name: matterbridge-linux-64bit
+        path: output/lin
+    - name: Upload windows 64-bit
+      if: startsWith(matrix.go-version,'1.16')
+      uses: actions/upload-artifact@v2
+      with:
+        name: matterbridge-windows-64bit
+        path: output/win
+    - name: Upload darwin 64-bit
+      if: startsWith(matrix.go-version,'1.16')
+      uses: actions/upload-artifact@v2
+      with:
+        name: matterbridge-darwin-64bit
+        path: output/mac

.golangci.yaml

@@ -23,7 +23,7 @@ run:
   # default value is empty list, but next dirs are always skipped independently
   # from this option's value:
   #   	vendor$, third_party$, testdata$, examples$, Godeps$, builtin$
-  skip-dirs:
+  skip-dirs: gateway/bridgemap$
 
   # which files to skip: they will be analyzed, but issues from them
   # won't be reported. Default value is empty list, but there is
@@ -91,7 +91,6 @@ linters-settings:
     # Correct spellings using locale preferences for US or UK.
     # Default is to use a neutral variety of English.
     # Setting locale to US will correct the British spelling of 'colour' to 'color'.
-    locale: US
   lll:
     # max line length, lines longer will be reported. Default is 120.
     # '\t' is counted as 1 character by default, and can be changed with the tab-width option
@@ -176,7 +175,13 @@ linters:
     - prealloc
     - wsl
     - gomnd
-
+    - godox
+    - goerr113
+    - testpackage
+    - godot
+    - interfacer
+    - goheader
+    - noctx
 
 # rules to deal with reported isues
 issues:

.travis.yml

@@ -1,56 +0,0 @@
-language: go
-go_import_path: github.com/42wim/matterbridge
-
-# We have everything vendored so this helps TravisCI not run `go get ...`.
-install: true
-
-git:
-  depth: 200
-
-notifications:
-  email: false
-
-branches:
-  only:
-  - master
-  - /.*/
-
-jobs:
-  include:
-  - stage: lint
-    # Run linting in one Go environment only.
-    script: ./ci/lint.sh
-    go: 1.14.x
-    env:
-    - GO111MODULE=on
-    - GOLANGCI_VERSION="v1.23.7"
-  - stage: test
-    # Run tests in a combination of Go environments.
-    script: ./ci/test.sh
-    go: 1.13.x
-    env:
-    - GOFLAGS=-mod=vendor
-  - script: ./ci/test.sh
-    go: 1.13.x
-    env:
-    - GO111MODULE=on
-  - script: ./ci/test.sh
-    go: 1.14.x
-    env:
-    - GO111MODULE=on
-    - REPORT_COVERAGE=1
-    - BINDEPLOY=1
-
-before_deploy: /bin/bash ci/bintray.sh
-
-deploy:
-  on:
-    all_branches: true
-    condition: $BINDEPLOY = 1
-  provider: bintray
-  edge:
-    branch: v1.8.47
-  file: ci/deploy.json
-  user: 42wim
-  key:
-    secure: "CeXXe6JOmt7HYR81MdWLua0ltQHhDdkIeRGBFbgd7hkb1wi8eF9DgpAcQrTso8NIlHNZmSAP46uhFgsRvkuezzX0ygalZ7DCJyAyn3sAMEh+UQSHV1WGThRehTtidqRGjetzsIGSwdrJOWil+XTfbO1Z8DGzfakhSuAZka8CM4BAoe3YeP9rYK8h+84x0GHfczvsLtXZ3mWLvQuwe4pK6+ItBCUg0ae7O7ZUpWHy0xQQkkWztY/6RAzXfaG7DuGjIw+20fhx3WOXRNpHCtZ6Bc3qERCpk0s1HhlQWlrN9wDaFTBWYwlvSnNgvxxMbNXJ6RrRJ0l0bA7FUswYwyroxhzrGLdzWDg8dHaQkypocngdalfhpsnoO9j3ApJhomUFJ3UoEq5nOGRUrKn8MPi+dP0zE4kNQ3e4VNa1ufNrvfpWolMg3xh8OXuhQdD5wIM5zFAbRJLqWSCVAjPq4DDPecmvXBOlIial7oa312lN5qnBnUjvAcxszZ+FUyDHT1Grxzna4tMwxY9obPzZUzm7359AOCCwIQFVB8GLqD2nwIstcXS0zGRz+fhviPipHuBa02q5bGUZwmkvrSNab0s8Jo7pCrel2Rz3nWPKaiCfq2WjbW1CLheSMkOQrjsdUd1hhbqNWFPUjJPInTc77NAKCfm5runv5uyowRLh4NNd0sI="

Dockerfile

@@ -1,11 +1,14 @@
-FROM alpine:edge
-ENTRYPOINT ["/bin/matterbridge"]
+FROM alpine AS builder
 
-COPY . /go/src/github.com/42wim/matterbridge
-RUN apk update && apk add go git gcc musl-dev ca-certificates mailcap \
-        && cd /go/src/github.com/42wim/matterbridge \
-        && export GOPATH=/go \
-        && go get \
-        && go build -x -ldflags "-X main.githash=$(git log --pretty=format:'%h' -n 1)" -o /bin/matterbridge \
-        && rm -rf /go \
-        && apk del --purge git go gcc musl-dev
+COPY . /go/src/matterbridge
+RUN apk --no-cache add go git \
+        && cd /go/src/matterbridge \
+        && go build -mod vendor -ldflags "-X main.githash=$(git log --pretty=format:'%h' -n 1)" -o /bin/matterbridge
+
+FROM alpine
+RUN apk --no-cache add ca-certificates mailcap
+COPY --from=builder /bin/matterbridge /bin/matterbridge
+RUN mkdir /etc/matterbridge \
+  && touch /etc/matterbridge/matterbridge.toml \
+  && ln -sf /matterbridge.toml /etc/matterbridge/matterbridge.toml
+ENTRYPOINT ["/bin/matterbridge", "-conf", "/etc/matterbridge/matterbridge.toml"]

README.md

@@ -9,26 +9,26 @@ Letting people be where they want to be.<br />
 
    <sup>
 
+[Discord][mb-discord] |
 [Gitter][mb-gitter] |
 [IRC][mb-irc] |
-[Discord][mb-discord] |
+[Keybase][mb-keybase] |
 [Matrix][mb-matrix] |
-[Slack][mb-slack] |
 [Mattermost][mb-mattermost] |
+[MSTeams][mb-msteams] |
 [Rocket.Chat][mb-rocketchat] |
-[XMPP][mb-xmpp] |
+[Slack][mb-slack] |
+[Telegram][mb-telegram] |
 [Twitch][mb-twitch] |
 [WhatsApp][mb-whatsapp] |
+[XMPP][mb-xmpp] |
 [Zulip][mb-zulip] |
-[Telegram][mb-telegram] |
-[Keybase][mb-keybase] |
 And more...
 </sup>
 
 ---
 
 [![Download stable](https://img.shields.io/github/release/42wim/matterbridge.svg?label=download%20stable)](https://github.com/42wim/matterbridge/releases/latest)
-[![Download dev](https://img.shields.io/bintray/v/42wim/nightly/Matterbridge.svg?label=download%20dev&colorB=007ec6)](https://bintray.com/42wim/nightly/Matterbridge/_latestVersion)
 [![Maintainability](https://api.codeclimate.com/v1/badges/82dff70ef2ba85a6173a/maintainability)](https://codeclimate.com/github/42wim/matterbridge/maintainability)
 [![Test Coverage](https://api.codeclimate.com/v1/badges/82dff70ef2ba85a6173a/test_coverage)](https://codeclimate.com/github/42wim/matterbridge/test_coverage)<br />
 
@@ -70,7 +70,7 @@ And more...
   - [Changelog](#changelog)
   - [FAQ](#faq)
   - [Related projects](#related-projects)
-  - [Articles](#articles)
+  - [Articles / Tutorials](#articles--tutorials)
   - [Thanks](#thanks)
 
 ## Features
@@ -86,29 +86,39 @@ And more...
 
 ### Natively supported
 
-- [Mattermost](https://github.com/mattermost/mattermost-server/) 4.x, 5.x
-- [IRC](http://www.mirc.com/servers.html)
-- [XMPP](https://xmpp.org)
-- [Gitter](https://gitter.im)
-- [Slack](https://slack.com)
 - [Discord](https://discordapp.com)
-- [Telegram](https://telegram.org)
-- [Rocket.chat](https://rocket.chat)
-- [Matrix](https://matrix.org)
-- [Steam](https://store.steampowered.com/)
-- [Twitch](https://twitch.tv)
-- [Ssh-chat](https://github.com/shazow/ssh-chat)
-- [WhatsApp](https://www.whatsapp.com/)
-- [Zulip](https://zulipchat.com)
+- [Gitter](https://gitter.im)
+- [IRC](http://www.mirc.com/servers.html)
 - [Keybase](https://keybase.io)
+- [Matrix](https://matrix.org)
+- [Mattermost](https://github.com/mattermost/mattermost-server/)
+- [Microsoft Teams](https://teams.microsoft.com)
+- [Mumble](https://www.mumble.info/)
+- [Nextcloud Talk](https://nextcloud.com/talk/)
+- [Rocket.chat](https://rocket.chat)
+- [Slack](https://slack.com)
+- [Ssh-chat](https://github.com/shazow/ssh-chat)
+- ~~[Steam](https://store.steampowered.com/)~~
+  - Not supported anymore, see [here](https://github.com/Philipp15b/go-steam/issues/94) for more info.
+- [Telegram](https://telegram.org)
+- [Twitch](https://twitch.tv)
+- [VK](https://vk.com/)
+- [WhatsApp](https://www.whatsapp.com/)
+- [XMPP](https://xmpp.org)
+- [Zulip](https://zulipchat.com)
 
 ### 3rd party via matterbridge api
 
-- [Minecraft](https://github.com/elytra/MatterLink)
-- [Reddit](https://github.com/bonehurtingjuice/mattereddit)
-- [Facebook messenger](https://github.com/VictorNine/fbridge)
 - [Discourse](https://github.com/DeclanHoare/matterbabble)
+- [Facebook messenger](https://github.com/powerjungle/fbridge-asyncio)
+- [Facebook messenger](https://github.com/VictorNine/fbridge)
+- [Minecraft](https://github.com/elytra/MatterLink)
+- [Minecraft](https://github.com/raws/mattercraft)
+- [Minecraft](https://gitlab.com/Programie/MatterBukkit)
+- [Reddit](https://github.com/bonehurtingjuice/mattereddit)
 - [Counter-Strike, half-life and more](https://forums.alliedmods.net/showthread.php?t=319430)
+- [MatterAMXX](https://github.com/GabeIggy/MatterAMXX)
+- [Vintage Story](https://github.com/NikkyAI/vs-matterbridge)
 
 ### API
 
@@ -117,62 +127,67 @@ More info and examples on the [wiki](https://github.com/42wim/matterbridge/wiki/
 
 Used by the projects below. Feel free to make a PR to add your project to this list.
 
-- [MatterLink](https://github.com/elytra/MatterLink) (Matterbridge link for Minecraft Server chat)
+- [MatterLink](https://github.com/elytra/MatterLink) (Matterbridge link for Minecraft Forge server chat, archived)
+- [MatterCraft](https://github.com/raws/mattercraft) (Matterbridge link for Minecraft Forge server chat)
+- [MatterBukkit](https://gitlab.com/Programie/MatterBukkit) (Matterbridge link for Minecraft Bukkit/Spigot server chat)
 - [pyCord](https://github.com/NikkyAI/pyCord) (crossplatform chatbot)
 - [Mattereddit](https://github.com/bonehurtingjuice/mattereddit) (Reddit chat support)
+- [fbridge-asyncio](https://github.com/powerjungle/fbridge-asyncio) (Facebook messenger support)
 - [fbridge](https://github.com/VictorNine/fbridge) (Facebook messenger support)
 - [matterbabble](https://github.com/DeclanHoare/matterbabble) (Discourse support)
 - [MatterAMXX](https://forums.alliedmods.net/showthread.php?t=319430) (Counter-Strike, half-life and more via AMXX mod)
+- [Vintage Story](https://github.com/NikkyAI/vs-matterbridge)
 
 ## Chat with us
 
 Questions or want to test on your favorite platform? Join below:
 
+- [Discord][mb-discord]
 - [Gitter][mb-gitter]
 - [IRC][mb-irc]
-- [Discord][mb-discord]
+- [Keybase][mb-keybase]
 - [Matrix][mb-matrix]
-- [Slack][mb-slack]
 - [Mattermost][mb-mattermost]
 - [Rocket.Chat][mb-rocketchat]
-- [XMPP][mb-xmpp] (matterbridge@conference.jabber.de)
-- [Twitch][mb-twitch]
-- [Zulip][mb-zulip]
+- [Slack][mb-slack]
 - [Telegram][mb-telegram]
-- [Keybase][mb-keybase]
+- [Twitch][mb-twitch]
+- [XMPP][mb-xmpp] (matterbridge@conference.jabber.de)
+- [Zulip][mb-zulip]
 
 ## Screenshots
 
-See https://github.com/42wim/matterbridge/wiki
+See <https://github.com/42wim/matterbridge/wiki>
 
 ## Installing / upgrading
 
 ### Binaries
 
-- Latest stable release [v1.17.0](https://github.com/42wim/matterbridge/releases/latest)
-- Development releases (follows master) can be downloaded [here](https://dl.bintray.com/42wim/nightly/)
+- Latest stable release [v1.22.2](https://github.com/42wim/matterbridge/releases/latest)
+- Development releases (follows master) can be downloaded [here](https://github.com/42wim/matterbridge/actions) selecting the latest green build and then artifacts.
 
-To install or upgrade just download the latest [binary](https://github.com/42wim/matterbridge/releases/latest) and follow the instructions on the [howto](https://github.com/42wim/matterbridge/wiki/How-to-create-your-config) for a step by step walkthrough for creating your configuration.
+To install or upgrade just download the latest [binary](https://github.com/42wim/matterbridge/releases/latest). On \*nix platforms you may need to make the binary executable - you can do this by running `chmod a+x` on the binary (example: `chmod a+x matterbridge-1.20.0-linux-64bit`). After downloading (and making the binary executable, if necessary), follow the instructions on the [howto](https://github.com/42wim/matterbridge/wiki/How-to-create-your-config) for a step by step walkthrough for creating your configuration.
 
 ### Packages
 
 - [Overview](https://repology.org/metapackage/matterbridge/versions)
+- [snap](https://snapcraft.io/matterbridge)
+- [scoop](https://github.com/42wim/scoop-bucket)
 
 ## Building
 
 Most people just want to use binaries, you can find those [here](https://github.com/42wim/matterbridge/releases/latest)
 
 If you really want to build from source, follow these instructions:
-Go 1.12+ is required. Make sure you have [Go](https://golang.org/doc/install) properly installed.
+Go 1.13+ is required. Make sure you have [Go](https://golang.org/doc/install) properly installed.
 
-
-```
+```bash
 go get github.com/42wim/matterbridge
 ```
 
 You should now have matterbridge binary in the ~/go/bin directory:
 
-```
+```bash
 $ ls ~/go/bin/
 matterbridge
 ```
@@ -197,8 +212,8 @@ All possible [settings](https://github.com/42wim/matterbridge/wiki/Settings) for
 
 ```toml
 [irc]
-    [irc.freenode]
-    Server="irc.freenode.net:6667"
+    [irc.libera]
+    Server="irc.libera.chat:6667"
     Nick="yourbotname"
 
 [mattermost]
@@ -214,7 +229,7 @@ All possible [settings](https://github.com/42wim/matterbridge/wiki/Settings) for
 name="mygateway"
 enable=true
     [[gateway.inout]]
-    account="irc.freenode"
+    account="irc.libera"
     channel="#testing"
 
     [[gateway.inout]]
@@ -255,7 +270,7 @@ RemoteNickFormat="[{PROTOCOL}/{BRIDGE}] <{NICK}> "
 
 See [howto](https://github.com/42wim/matterbridge/wiki/How-to-create-your-config) for a step by step walkthrough for creating your configuration.
 
-```
+```bash
 Usage of ./matterbridge:
   -conf string
         config file (default "matterbridge.toml")
@@ -292,18 +307,23 @@ See [FAQ](https://github.com/42wim/matterbridge/wiki/FAQ)
 - [fbridge](https://github.com/VictorNine/fbridge) (Facebook messenger support)
 - [isla](https://github.com/alphachung/isla) (Bot for Discord-Telegram groups used alongside matterbridge)
 - [matterbabble](https://github.com/DeclanHoare/matterbabble) (Connect Discourse threads to Matterbridge)
+- [nextcloud talk](https://github.com/nextcloud/talk_matterbridge) (Integrates matterbridge in Nextcloud Talk)
+- [mattercraft](https://github.com/raws/mattercraft) (Minecraft bridge)
+- [vs-matterbridge](https://github.com/NikkyAI/vs-matterbridge) (Vintage Story bridge)
 
-## Articles
+## Articles / Tutorials
 
 - [matterbridge on kubernetes](https://medium.freecodecamp.org/using-kubernetes-to-deploy-a-chat-gateway-or-when-technology-works-like-its-supposed-to-a169a8cd69a3)
-- https://mattermost.com/blog/connect-irc-to-mattermost/
-- https://blog.valvin.fr/2016/09/17/mattermost-et-un-channel-irc-cest-possible/
-- https://blog.brightscout.com/top-10-mattermost-integrations/
-- http://bencey.co.nz/2018/09/17/bridge/
-- https://www.algoo.fr/blog/2018/01/19/recouvrez-votre-liberte-en-quittant-slack-pour-un-mattermost-auto-heberge/
-- https://kopano.com/blog/matterbridge-bridging-mattermost-chat/
-- https://www.stitcher.com/s/?eid=52382713
-- https://daniele.tech/2019/02/how-to-use-matterbridge-to-connect-2-different-slack-workspaces/
+- <https://mattermost.com/blog/connect-irc-to-mattermost/>
+- <https://blog.valvin.fr/2016/09/17/mattermost-et-un-channel-irc-cest-possible/>
+- <https://blog.brightscout.com/top-10-mattermost-integrations/>
+- <https://www.algoo.fr/blog/2018/01/19/recouvrez-votre-liberte-en-quittant-slack-pour-un-mattermost-auto-heberge/>
+- <https://kopano.com/blog/matterbridge-bridging-mattermost-chat/>
+- <https://www.stitcher.com/s/?eid=52382713>
+- <https://daniele.tech/2019/02/how-to-use-matterbridge-to-connect-2-different-slack-workspaces/>
+- <https://userlinux.net/mattermost-and-matterbridge.html>
+- <https://nextcloud.com/blog/bridging-chat-services-in-talk/>
+- Youtube: [whatsapp - telegram bridging](https://www.youtube.com/watch?v=W-VXISoKtNc)
 
 ## Thanks
 
@@ -316,36 +336,42 @@ See [FAQ](https://github.com/42wim/matterbridge/wiki/FAQ)
 
 Matterbridge wouldn't exist without these libraries:
 
-- discord - https://github.com/bwmarrin/discordgo
-- echo - https://github.com/labstack/echo
-- gitter - https://github.com/sromku/go-gitter
-- gops - https://github.com/google/gops
-- gozulipbot - https://github.com/ifo/gozulipbot
-- irc - https://github.com/lrstanley/girc
-- mattermost - https://github.com/mattermost/mattermost-server
-- matrix - https://github.com/matrix-org/gomatrix
-- sshchat - https://github.com/shazow/ssh-chat
-- slack - https://github.com/nlopes/slack
-- steam - https://github.com/Philipp15b/go-steam
-- telegram - https://github.com/go-telegram-bot-api/telegram-bot-api
-- xmpp - https://github.com/mattn/go-xmpp
-- whatsapp - https://github.com/Rhymen/go-whatsapp/
-- zulip - https://github.com/ifo/gozulipbot
-- tengo - https://github.com/d5/tengo
-- keybase - https://github.com/keybase/go-keybase-chat-bot
+- discord - <https://github.com/bwmarrin/discordgo>
+- echo - <https://github.com/labstack/echo>
+- gitter - <https://github.com/sromku/go-gitter>
+- gops - <https://github.com/google/gops>
+- gozulipbot - <https://github.com/ifo/gozulipbot>
+- gumble - <https://github.com/layeh/gumble>
+- irc - <https://github.com/lrstanley/girc>
+- keybase - <https://github.com/keybase/go-keybase-chat-bot>
+- matrix - <https://github.com/matrix-org/gomatrix>
+- mattermost - <https://github.com/mattermost/mattermost-server>
+- msgraph.go - <https://github.com/yaegashi/msgraph.go>
+- mumble - <https://github.com/layeh/gumble>
+- nctalk - <https://github.com/gary-kim/go-nc-talk>
+- slack - <https://github.com/nlopes/slack>
+- sshchat - <https://github.com/shazow/ssh-chat>
+- steam - <https://github.com/Philipp15b/go-steam>
+- telegram - <https://github.com/go-telegram-bot-api/telegram-bot-api>
+- tengo - <https://github.com/d5/tengo>
+- vk - <https://github.com/SevereCloud/vksdk>
+- whatsapp - <https://github.com/Rhymen/go-whatsapp>
+- xmpp - <https://github.com/mattn/go-xmpp>
+- zulip - <https://github.com/ifo/gozulipbot>
 
 <!-- Links -->
 
-[mb-gitter]: https://gitter.im/42wim/matterbridge
-[mb-irc]: https://webchat.freenode.net/?channels=matterbridgechat
 [mb-discord]: https://discord.gg/AkKPtrQ
+[mb-gitter]: https://gitter.im/42wim/matterbridge
+[mb-irc]: https://web.libera.chat/#matterbridge
+[mb-keybase]: https://keybase.io/team/matterbridge
 [mb-matrix]: https://riot.im/app/#/room/#matterbridge:matrix.org
-[mb-slack]: https://join.slack.com/matterbridgechat/shared_invite/MjEwODMxNjU1NDMwLTE0OTk2MTU3NTMtMzZkZmRiNDZhOA
 [mb-mattermost]: https://framateam.org/signup_user_complete/?id=tfqm33ggop8x3qgu4boeieta6e
+[mb-msteams]: https://teams.microsoft.com/join/hj92x75gd3y7
 [mb-rocketchat]: https://open.rocket.chat/channel/matterbridge
-[mb-xmpp]: https://inverse.chat/
+[mb-slack]: https://join.slack.com/t/matterbridgechat/shared_invite/zt-2ourq2h2-7YvyYBq2WFGC~~zEzA68_Q
+[mb-telegram]: https://t.me/Matterbridge
 [mb-twitch]: https://www.twitch.tv/matterbridge
 [mb-whatsapp]: https://www.whatsapp.com/
-[mb-keybase]: https://keybase.io/team/matterbridge
+[mb-xmpp]: https://inverse.chat/
 [mb-zulip]: https://matterbridge.zulipchat.com/register/
-[mb-telegram]: https://t.me/Matterbridge

bridge/api/api.go

@@ -6,17 +6,20 @@ import (
 	"sync"
 	"time"
 
+	"gopkg.in/olahol/melody.v1"
+
 	"github.com/42wim/matterbridge/bridge"
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/labstack/echo/v4"
 	"github.com/labstack/echo/v4/middleware"
-	"github.com/zfjagann/golang-ring"
+	ring "github.com/zfjagann/golang-ring"
 )
 
 type API struct {
 	Messages ring.Ring
 	sync.RWMutex
 	*bridge.Config
+	mrouter *melody.Melody
 }
 
 type Message struct {
@@ -32,6 +35,32 @@ func New(cfg *bridge.Config) bridge.Bridger {
 	e := echo.New()
 	e.HideBanner = true
 	e.HidePort = true
+
+	b.mrouter = melody.New()
+	b.mrouter.HandleMessage(func(s *melody.Session, msg []byte) {
+		message := config.Message{}
+		err := json.Unmarshal(msg, &message)
+		if err != nil {
+			b.Log.Errorf("failed to decode message from byte[] '%s'", string(msg))
+			return
+		}
+		b.handleWebsocketMessage(message, s)
+	})
+	b.mrouter.HandleConnect(func(session *melody.Session) {
+		greet := b.getGreeting()
+		data, err := json.Marshal(greet)
+		if err != nil {
+			b.Log.Errorf("failed to encode message '%v'", greet)
+			return
+		}
+		err = session.Write(data)
+		if err != nil {
+			b.Log.Errorf("failed to write message '%s'", string(data))
+			return
+		}
+		// TODO: send message history buffer from `b.Messages` here
+	})
+
 	b.Messages = ring.Ring{}
 	if b.GetInt("Buffer") != 0 {
 		b.Messages.SetCapacity(b.GetInt("Buffer"))
@@ -41,9 +70,17 @@ func New(cfg *bridge.Config) bridge.Bridger {
 			return key == b.GetString("Token"), nil
 		}))
 	}
+
+	// Set RemoteNickFormat to a sane default
+	if !b.IsKeySet("RemoteNickFormat") {
+		b.Log.Debugln("RemoteNickFormat is unset, defaulting to \"{NICK}\"")
+		b.Config.Config.Viper().Set(b.GetConfigKey("RemoteNickFormat"), "{NICK}")
+	}
+
 	e.GET("/api/health", b.handleHealthcheck)
 	e.GET("/api/messages", b.handleMessages)
 	e.GET("/api/stream", b.handleStream)
+	e.GET("/api/websocket", b.handleWebsocket)
 	e.POST("/api/message", b.handlePostMessage)
 	go func() {
 		if b.GetString("BindAddress") == "" {
@@ -58,13 +95,13 @@ func New(cfg *bridge.Config) bridge.Bridger {
 func (b *API) Connect() error {
 	return nil
 }
+
 func (b *API) Disconnect() error {
 	return nil
-
 }
+
 func (b *API) JoinChannel(channel config.ChannelInfo) error {
 	return nil
-
 }
 
 func (b *API) Send(msg config.Message) (string, error) {
@@ -74,7 +111,14 @@ func (b *API) Send(msg config.Message) (string, error) {
 	if msg.Event == config.EventMsgDelete {
 		return "", nil
 	}
-	b.Messages.Enqueue(&msg)
+	b.Log.Debugf("enqueueing message from %s on ring buffer", msg.Username)
+	b.Messages.Enqueue(msg)
+
+	data, err := json.Marshal(msg)
+	if err != nil {
+		b.Log.Errorf("failed to encode message  '%s'", msg)
+	}
+	_ = b.mrouter.Broadcast(data)
 	return "", nil
 }
 
@@ -106,18 +150,23 @@ func (b *API) handleMessages(c echo.Context) error {
 	return nil
 }
 
-func (b *API) handleStream(c echo.Context) error {
-	c.Response().Header().Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
-	c.Response().WriteHeader(http.StatusOK)
-	greet := config.Message{
+func (b *API) getGreeting() config.Message {
+	return config.Message{
 		Event:     config.EventAPIConnected,
 		Timestamp: time.Now(),
 	}
+}
+
+func (b *API) handleStream(c echo.Context) error {
+	c.Response().Header().Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
+	c.Response().WriteHeader(http.StatusOK)
+	greet := b.getGreeting()
 	if err := json.NewEncoder(c.Response()).Encode(greet); err != nil {
 		return err
 	}
 	c.Response().Flush()
 	for {
+		// TODO: this causes issues, messages should be broadcasted to all connected clients
 		msg := b.Messages.Dequeue()
 		if msg != nil {
 			if err := json.NewEncoder(c.Response()).Encode(msg); err != nil {
@@ -128,3 +177,31 @@ func (b *API) handleStream(c echo.Context) error {
 		time.Sleep(200 * time.Millisecond)
 	}
 }
+
+func (b *API) handleWebsocketMessage(message config.Message, s *melody.Session) {
+	message.Channel = "api"
+	message.Protocol = "api"
+	message.Account = b.Account
+	message.ID = ""
+	message.Timestamp = time.Now()
+
+	data, err := json.Marshal(message)
+	if err != nil {
+		b.Log.Errorf("failed to encode message for loopback '%v'", message)
+		return
+	}
+	_ = b.mrouter.BroadcastOthers(data, s)
+
+	b.Log.Debugf("Sending websocket message from %s on %s to gateway", message.Username, "api")
+	b.Remote <- message
+}
+
+func (b *API) handleWebsocket(c echo.Context) error {
+	err := b.mrouter.HandleRequest(c.Response(), c.Request())
+	if err != nil {
+		b.Log.Errorf("error in websocket handling  '%v'", err)
+		return err
+	}
+
+	return nil
+}

bridge/bridge.go

@@ -4,6 +4,7 @@ import (
 	"log"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/sirupsen/logrus"
@@ -74,6 +75,7 @@ func (b *Bridge) joinChannels(channels map[string]config.ChannelInfo, exists map
 	for ID, channel := range channels {
 		if !exists[ID] {
 			b.Log.Infof("%s: joining %s (ID: %s)", b.Account, channel.Name, ID)
+			time.Sleep(time.Duration(b.GetInt("JoinDelay")) * time.Millisecond)
 			err := b.JoinChannel(channel)
 			if err != nil {
 				return err
@@ -84,8 +86,16 @@ func (b *Bridge) joinChannels(channels map[string]config.ChannelInfo, exists map
 	return nil
 }
 
+func (b *Bridge) GetConfigKey(key string) string {
+	return b.Account + "." + key
+}
+
+func (b *Bridge) IsKeySet(key string) bool {
+	return b.Config.IsKeySet(b.GetConfigKey(key)) || b.Config.IsKeySet("general."+key)
+}
+
 func (b *Bridge) GetBool(key string) bool {
-	val, ok := b.Config.GetBool(b.Account + "." + key)
+	val, ok := b.Config.GetBool(b.GetConfigKey(key))
 	if !ok {
 		val, _ = b.Config.GetBool("general." + key)
 	}
@@ -93,7 +103,7 @@ func (b *Bridge) GetBool(key string) bool {
 }
 
 func (b *Bridge) GetInt(key string) int {
-	val, ok := b.Config.GetInt(b.Account + "." + key)
+	val, ok := b.Config.GetInt(b.GetConfigKey(key))
 	if !ok {
 		val, _ = b.Config.GetInt("general." + key)
 	}
@@ -101,7 +111,7 @@ func (b *Bridge) GetInt(key string) int {
 }
 
 func (b *Bridge) GetString(key string) string {
-	val, ok := b.Config.GetString(b.Account + "." + key)
+	val, ok := b.Config.GetString(b.GetConfigKey(key))
 	if !ok {
 		val, _ = b.Config.GetString("general." + key)
 	}
@@ -109,7 +119,7 @@ func (b *Bridge) GetString(key string) string {
 }
 
 func (b *Bridge) GetStringSlice(key string) []string {
-	val, ok := b.Config.GetStringSlice(b.Account + "." + key)
+	val, ok := b.Config.GetStringSlice(b.GetConfigKey(key))
 	if !ok {
 		val, _ = b.Config.GetStringSlice("general." + key)
 	}
@@ -117,7 +127,7 @@ func (b *Bridge) GetStringSlice(key string) []string {
 }
 
 func (b *Bridge) GetStringSlice2D(key string) [][]string {
-	val, ok := b.Config.GetStringSlice2D(b.Account + "." + key)
+	val, ok := b.Config.GetStringSlice2D(b.GetConfigKey(key))
 	if !ok {
 		val, _ = b.Config.GetStringSlice2D("general." + key)
 	}

bridge/config/config.go

@@ -3,6 +3,7 @@ package config
 import (
 	"bytes"
 	"io/ioutil"
+	"os"
 	"path/filepath"
 	"strings"
 	"sync"
@@ -25,8 +26,11 @@ const (
 	EventAPIConnected      = "api_connected"
 	EventUserTyping        = "user_typing"
 	EventGetChannelMembers = "get_channel_members"
+	EventNoticeIRC         = "notice_irc"
 )
 
+const ParentIDNotFound = "msg-parent-not-found"
+
 type Message struct {
 	Text      string    `json:"text"`
 	Channel   string    `json:"channel"`
@@ -43,6 +47,14 @@ type Message struct {
 	Extra     map[string][]interface{}
 }
 
+func (m Message) ParentNotFound() bool {
+	return m.ParentID == ParentIDNotFound
+}
+
+func (m Message) ParentValid() bool {
+	return m.ParentID != "" && !m.ParentNotFound()
+}
+
 type FileInfo struct {
 	Name    string
 	Data    *[]byte
@@ -73,29 +85,34 @@ type ChannelMember struct {
 type ChannelMembers []ChannelMember
 
 type Protocol struct {
-	AuthCode               string // steam
-	BindAddress            string // mattermost, slack // DEPRECATED
-	Buffer                 int    // api
-	Charset                string // irc
-	ClientID               string // msteams
-	ColorNicks             bool   // only irc for now
-	Debug                  bool   // general
-	DebugLevel             int    // only for irc now
-	DisableWebPagePreview  bool   // telegram
-	EditSuffix             string // mattermost, slack, discord, telegram, gitter
-	EditDisable            bool   // mattermost, slack, discord, telegram, gitter
-	IconURL                string // mattermost, slack
-	IgnoreFailureOnStart   bool   // general
-	IgnoreNicks            string // all protocols
-	IgnoreMessages         string // all protocols
-	Jid                    string // xmpp
-	Label                  string // all protocols
-	Login                  string // mattermost, matrix
+	AllowMention           []string // discord
+	AuthCode               string   // steam
+	BindAddress            string   // mattermost, slack // DEPRECATED
+	Buffer                 int      // api
+	Charset                string   // irc
+	ClientID               string   // msteams
+	ColorNicks             bool     // only irc for now
+	Debug                  bool     // general
+	DebugLevel             int      // only for irc now
+	DisableWebPagePreview  bool     // telegram
+	EditSuffix             string   // mattermost, slack, discord, telegram, gitter
+	EditDisable            bool     // mattermost, slack, discord, telegram, gitter
+	HTMLDisable            bool     // matrix
+	IconURL                string   // mattermost, slack
+	IgnoreFailureOnStart   bool     // general
+	IgnoreNicks            string   // all protocols
+	IgnoreMessages         string   // all protocols
+	Jid                    string   // xmpp
+	JoinDelay              string   // all protocols
+	Label                  string   // all protocols
+	Login                  string   // mattermost, matrix
+	LogFile                string   // general
 	MediaDownloadBlackList []string
 	MediaDownloadPath      string // Basically MediaServerUpload, but instead of uploading it, just write it to a file on the same server.
 	MediaDownloadSize      int    // all protocols
 	MediaServerDownload    string
 	MediaServerUpload      string
+	MediaConvertTgs        string     // telegram
 	MediaConvertWebPToPNG  bool       // telegram
 	MessageDelay           int        // IRC, time in millisecond to wait between messages
 	MessageFormat          string     // telegram
@@ -103,6 +120,7 @@ type Protocol struct {
 	MessageQueue           int        // IRC, size of message queue for flood control
 	MessageSplit           bool       // IRC, split long messages with newlines on MessageLength instead of clipping
 	Muc                    string     // xmpp
+	MxID                   string     // matrix
 	Name                   string     // all protocols
 	Nick                   string     // all protocols
 	NickFormatter          string     // mattermost, slack
@@ -112,7 +130,7 @@ type Protocol struct {
 	NicksPerRow            int        // mattermost, slack
 	NoHomeServerSuffix     bool       // matrix
 	NoSendJoinPart         bool       // all protocols
-	NoTLS                  bool       // mattermost
+	NoTLS                  bool       // mattermost, xmpp
 	Password               string     // IRC,mattermost,XMPP,matrix
 	PrefixMessagesWithNick bool       // mattemost, slack
 	PreserveThreading      bool       // slack
@@ -125,7 +143,7 @@ type Protocol struct {
 	ReplaceNicks           [][]string // all protocols
 	RemoteNickFormat       string     // all protocols
 	RunCommands            []string   // IRC
-	Server                 string     // IRC,mattermost,XMPP,discord
+	Server                 string     // IRC,mattermost,XMPP,discord,matrix
 	SessionFile            string     // msteams,whatsapp
 	ShowJoinPart           bool       // all protocols
 	ShowTopicChange        bool       // slack
@@ -134,12 +152,13 @@ type Protocol struct {
 	SkipTLSVerify          bool       // IRC, mattermost
 	SkipVersionCheck       bool       // mattermost
 	StripNick              bool       // all protocols
+	StripMarkdown          bool       // irc
 	SyncTopic              bool       // slack
 	TengoModifyMessage     string     // general
 	Team                   string     // mattermost, keybase
 	TeamID                 string     // msteams
 	TenantID               string     // msteams
-	Token                  string     // gitter, slack, discord, api
+	Token                  string     // gitter, slack, discord, api, matrix
 	Topic                  string     // zulip
 	URL                    string     // mattermost, slack // DEPRECATED
 	UseAPI                 bool       // mattermost, slack
@@ -148,7 +167,7 @@ type Protocol struct {
 	UseTLS                 bool       // IRC
 	UseDiscriminator       bool       // discord
 	UseFirstName           bool       // telegram
-	UseUserName            bool       // discord
+	UseUserName            bool       // discord, matrix
 	UseInsecureURL         bool       // telegram
 	VerboseJoinPart        bool       // IRC
 	WebhookBindAddress     string     // mattermost, slack
@@ -207,6 +226,7 @@ type BridgeValues struct {
 	WhatsApp           map[string]Protocol // TODO is this struct used? Search for "SlackLegacy" for example didn't return any results
 	Zulip              map[string]Protocol
 	Keybase            map[string]Protocol
+	Mumble             map[string]Protocol
 	General            Protocol
 	Tengo              Tengo
 	Gateway            []Gateway
@@ -216,6 +236,7 @@ type BridgeValues struct {
 type Config interface {
 	Viper() *viper.Viper
 	BridgeValues() *BridgeValues
+	IsKeySet(key string) bool
 	GetBool(key string) (bool, bool)
 	GetInt(key string) (int, bool)
 	GetString(key string) (string, bool)
@@ -243,6 +264,15 @@ func NewConfig(rootLogger *logrus.Logger, cfgfile string) Config {
 
 	cfgtype := detectConfigType(cfgfile)
 	mycfg := newConfigFromString(logger, input, cfgtype)
+	if mycfg.cv.General.LogFile != "" {
+		logfile, err := os.OpenFile(mycfg.cv.General.LogFile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600)
+		if err == nil {
+			logger.Info("Opening log file ", mycfg.cv.General.LogFile)
+			rootLogger.Out = logfile
+		} else {
+			logger.Warn("Failed to open ", mycfg.cv.General.LogFile)
+		}
+	}
 	if mycfg.cv.General.MediaDownloadSize == 0 {
 		mycfg.cv.General.MediaDownloadSize = 1000000
 	}
@@ -300,6 +330,12 @@ func (c *config) Viper() *viper.Viper {
 	return c.v
 }
 
+func (c *config) IsKeySet(key string) bool {
+	c.RLock()
+	defer c.RUnlock()
+	return c.v.IsSet(key)
+}
+
 func (c *config) GetBool(key string) (bool, bool) {
 	c.RLock()
 	defer c.RUnlock()
@@ -359,6 +395,11 @@ type TestConfig struct {
 	Overrides map[string]interface{}
 }
 
+func (c *TestConfig) IsKeySet(key string) bool {
+	_, ok := c.Overrides[key]
+	return ok || c.Config.IsKeySet(key)
+}
+
 func (c *TestConfig) GetBool(key string) (bool, bool) {
 	val, ok := c.Overrides[key]
 	if ok {

bridge/discord/discord.go

@@ -2,13 +2,13 @@ package bdiscord
 
 import (
 	"bytes"
-	"errors"
 	"fmt"
 	"strings"
 	"sync"
 
 	"github.com/42wim/matterbridge/bridge"
 	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/42wim/matterbridge/bridge/discord/transmitter"
 	"github.com/42wim/matterbridge/bridge/helper"
 	"github.com/matterbridge/discordgo"
 )
@@ -20,11 +20,9 @@ type Bdiscord struct {
 
 	c *discordgo.Session
 
-	nick            string
-	guildID         string
-	webhookID       string
-	webhookToken    string
-	canEditWebhooks bool
+	nick    string
+	userID  string
+	guildID string
 
 	channelsMutex  sync.RWMutex
 	channels       []*discordgo.Channel
@@ -33,6 +31,10 @@ type Bdiscord struct {
 	membersMutex  sync.RWMutex
 	userMemberMap map[string]*discordgo.Member
 	nickMemberMap map[string]*discordgo.Member
+
+	// Webhook specific logic
+	useAutoWebhooks bool
+	transmitter     *transmitter.Transmitter
 }
 
 func New(cfg *bridge.Config) bridge.Bridger {
@@ -40,23 +42,18 @@ func New(cfg *bridge.Config) bridge.Bridger {
 	b.userMemberMap = make(map[string]*discordgo.Member)
 	b.nickMemberMap = make(map[string]*discordgo.Member)
 	b.channelInfoMap = make(map[string]*config.ChannelInfo)
-	if b.GetString("WebhookURL") != "" {
-		b.Log.Debug("Configuring Discord Incoming Webhook")
-		b.webhookID, b.webhookToken = b.splitURL(b.GetString("WebhookURL"))
+
+	b.useAutoWebhooks = b.GetBool("AutoWebhooks")
+	if b.useAutoWebhooks {
+		b.Log.Debug("Using automatic webhooks")
 	}
 	return b
 }
 
 func (b *Bdiscord) Connect() error {
 	var err error
-	var guildFound bool
 	token := b.GetString("Token")
 	b.Log.Info("Connecting")
-	if b.GetString("WebhookURL") == "" {
-		b.Log.Info("Connecting using token")
-	} else {
-		b.Log.Info("Connecting using webhookurl (for posting) and token")
-	}
 	if !strings.HasPrefix(b.GetString("Token"), "Bot ") {
 		token = "Bot " + b.GetString("Token")
 	}
@@ -78,6 +75,11 @@ func (b *Bdiscord) Connect() error {
 	b.c.AddHandler(b.messageDeleteBulk)
 	b.c.AddHandler(b.memberAdd)
 	b.c.AddHandler(b.memberRemove)
+	// Add privileged intent for guild member tracking. This is needed to track nicks
+	// for display names and @mention translation
+	b.c.Identify.Intents = discordgo.MakeIntent(discordgo.IntentsAllWithoutPrivileged |
+		discordgo.IntentsGuildMembers)
+
 	err = b.c.Open()
 	if err != nil {
 		return err
@@ -92,62 +94,108 @@ func (b *Bdiscord) Connect() error {
 	}
 	serverName := strings.Replace(b.GetString("Server"), "ID:", "", -1)
 	b.nick = userinfo.Username
+	b.userID = userinfo.ID
+
+	// Try and find this account's guild, and populate channels
 	b.channelsMutex.Lock()
 	for _, guild := range guilds {
-		if guild.Name == serverName || guild.ID == serverName {
-			b.channels, err = b.c.GuildChannels(guild.ID)
-			if err != nil {
-				break
-			}
-			b.guildID = guild.ID
-			guildFound = true
+		// Skip, if the server name does not match the visible name or the ID
+		if guild.Name != serverName && guild.ID != serverName {
+			continue
 		}
+
+		// Complain about an ambiguous Server setting. Two Discord servers could have the same title!
+		// For IDs, practically this will never happen. It would only trigger if some server's name is also an ID.
+		if b.guildID != "" {
+			return fmt.Errorf("found multiple Discord servers with the same name %#v, expected to see only one", serverName)
+		}
+
+		// Getting this guild's channel could result in a permission error
+		b.channels, err = b.c.GuildChannels(guild.ID)
+		if err != nil {
+			return fmt.Errorf("could not get %#v's channels: %w", b.GetString("Server"), err)
+		}
+
+		b.guildID = guild.ID
 	}
 	b.channelsMutex.Unlock()
-	if !guildFound {
-		msg := fmt.Sprintf("Server \"%s\" not found", b.GetString("Server"))
-		err = errors.New(msg)
-		b.Log.Error(msg)
-		b.Log.Info("Possible values:")
+
+	// If we couldn't find a guild, we print extra debug information and return a nice error
+	if b.guildID == "" {
+		err = fmt.Errorf("could not find Discord server %#v", b.GetString("Server"))
+		b.Log.Error(err.Error())
+
+		// Print all of the possible server values
+		b.Log.Info("Possible server values:")
 		for _, guild := range guilds {
-			b.Log.Infof("Server=\"%s\" # Server name", guild.Name)
-			b.Log.Infof("Server=\"%s\" # Server ID", guild.ID)
+			b.Log.Infof("\t- Server=%#v # by name", guild.Name)
+			b.Log.Infof("\t- Server=%#v # by ID", guild.ID)
 		}
-	}
-	if err != nil {
+
+		// If there are no results, we should say that
+		if len(guilds) == 0 {
+			b.Log.Info("\t- (none found)")
+		}
+
 		return err
 	}
 
-	b.channelsMutex.RLock()
-	if b.GetString("WebhookURL") == "" {
-		for _, channel := range b.channels {
-			b.Log.Debugf("found channel %#v", channel)
-		}
-	} else {
-		manageWebhooks := discordgo.PermissionManageWebhooks
-		var channelsDenied []string
-		for _, info := range b.Channels {
-			id := b.getChannelID(info.Name) // note(qaisjp): this readlocks channelsMutex
-			b.Log.Debugf("Verifying PermissionManageWebhooks for %s with ID %s", info.ID, id)
+	// Legacy note: WebhookURL used to have an actual webhook URL that we would edit,
+	// but we stopped doing that due to Discord making rate limits more aggressive.
+	//
+	// Even older: the same WebhookURL used to be used by every channel, which is usually unexpected.
+	// This is no longer possible.
+	if b.GetString("WebhookURL") != "" {
+		message := "The global WebhookURL setting has been removed. "
+		message += "You can get similar \"webhook editing\" behaviour by replacing this line with `AutoWebhooks=true`. "
+		message += "If you rely on the old-OLD (non-editing) behaviour, can move the WebhookURL to specific channel sections."
+		b.Log.Errorln(message)
+		return fmt.Errorf("use of removed WebhookURL setting")
+	}
 
-			perms, permsErr := b.c.UserChannelPermissions(userinfo.ID, id)
-			if permsErr != nil {
-				b.Log.Warnf("Failed to check PermissionManageWebhooks in channel \"%s\": %s", info.Name, permsErr.Error())
-			} else if perms&manageWebhooks == manageWebhooks {
-				continue
+	if b.GetInt("debuglevel") > 0 {
+		b.Log.Debug("enabling even more discord debug")
+		b.c.Debug = true
+	}
+
+	// Initialise webhook management
+	b.transmitter = transmitter.New(b.c, b.guildID, "matterbridge", b.useAutoWebhooks)
+	b.transmitter.Log = b.Log
+
+	var webhookChannelIDs []string
+	for _, channel := range b.Channels {
+		channelID := b.getChannelID(channel.Name) // note(qaisjp): this readlocks channelsMutex
+
+		// If a WebhookURL was not explicitly provided for this channel,
+		// there are two options: just a regular bot message (ugly) or this is should be webhook sent
+		if channel.Options.WebhookURL == "" {
+			// If it should be webhook sent, we should enforce this via the transmitter
+			if b.useAutoWebhooks {
+				webhookChannelIDs = append(webhookChannelIDs, channelID)
 			}
-			channelsDenied = append(channelsDenied, fmt.Sprintf("%#v", info.Name))
+			continue
 		}
 
-		b.canEditWebhooks = len(channelsDenied) == 0
-		if b.canEditWebhooks {
-			b.Log.Info("Can manage webhooks; will edit channel for global webhook on send")
-		} else {
-			b.Log.Warn("Can't manage webhooks; won't edit channel for global webhook on send")
-			b.Log.Warn("Can't manage webhooks in channels: ", strings.Join(channelsDenied, ", "))
+		whID, whToken, ok := b.splitURL(channel.Options.WebhookURL)
+		if !ok {
+			return fmt.Errorf("failed to parse WebhookURL %#v for channel %#v", channel.Options.WebhookURL, channel.ID)
+		}
+
+		b.transmitter.AddWebhook(channelID, &discordgo.Webhook{
+			ID:        whID,
+			Token:     whToken,
+			GuildID:   b.guildID,
+			ChannelID: channelID,
+		})
+	}
+
+	if b.useAutoWebhooks {
+		err = b.transmitter.RefreshGuildWebhooks(webhookChannelIDs)
+		if err != nil {
+			b.Log.WithError(err).Println("transmitter could not refresh guild webhooks")
+			return err
 		}
 	}
-	b.channelsMutex.RUnlock()
 
 	// Obtaining guild members and initializing nickname mapping.
 	b.membersMutex.Lock()
@@ -204,76 +252,23 @@ func (b *Bdiscord) Send(msg config.Message) (string, error) {
 		msg.Text = "_" + msg.Text + "_"
 	}
 
-	// use initial webhook configured for the entire Discord account
-	isGlobalWebhook := true
-	wID := b.webhookID
-	wToken := b.webhookToken
-
-	// check if have a channel specific webhook
-	b.channelsMutex.RLock()
-	if ci, ok := b.channelInfoMap[msg.Channel+b.Account]; ok {
-		if ci.Options.WebhookURL != "" {
-			wID, wToken = b.splitURL(ci.Options.WebhookURL)
-			isGlobalWebhook = false
-		}
+	// Handle prefix hint for unthreaded messages.
+	if msg.ParentNotFound() {
+		msg.ParentID = ""
+		msg.Text = fmt.Sprintf("[thread]: %s", msg.Text)
 	}
-	b.channelsMutex.RUnlock()
 
 	// Use webhook to send the message
-	if wID != "" && msg.Event != config.EventMsgDelete {
-		// skip events
-		if msg.Event != "" && msg.Event != config.EventJoinLeave && msg.Event != config.EventTopicChange {
-			return "", nil
-		}
-
-		// If we are editing a message, delete the old message
-		if msg.ID != "" {
-			b.Log.Debugf("Deleting edited webhook message")
-			err := b.c.ChannelMessageDelete(channelID, msg.ID)
-			if err != nil {
-				b.Log.Errorf("Could not delete edited webhook message: %s", err)
-			}
-		}
-
-		b.Log.Debugf("Broadcasting using Webhook")
-
-		// skip empty messages
-		if msg.Text == "" && (msg.Extra == nil || len(msg.Extra["file"]) == 0) {
-			b.Log.Debugf("Skipping empty message %#v", msg)
-			return "", nil
-		}
-
-		msg.Text = helper.ClipMessage(msg.Text, MessageLength)
-		msg.Text = b.replaceUserMentions(msg.Text)
-		// discord username must be [0..32] max
-		if len(msg.Username) > 32 {
-			msg.Username = msg.Username[0:32]
-		}
-		// if we have a global webhook for this Discord account, and permission
-		// to modify webhooks (previously verified), then set its channel to
-		// the message channel before using it
-		// TODO: this isn't necessary if the last message from this webhook was
-		// sent to the current channel
-		if isGlobalWebhook && b.canEditWebhooks {
-			b.Log.Debugf("Setting webhook channel to \"%s\"", msg.Channel)
-			_, err := b.c.WebhookEdit(wID, "", "", channelID)
-			if err != nil {
-				b.Log.Errorf("Could not set webhook channel: %s", err)
-				return "", err
-			}
-		}
-		b.Log.Debugf("Processing webhook sending for message %#v", msg)
-		msg, err := b.webhookSend(&msg, wID, wToken)
-		if err != nil {
-			b.Log.Errorf("Could not broadcast via webook for message %#v: %s", msg, err)
-			return "", err
-		}
-		if msg == nil {
-			return "", nil
-		}
-		return msg.ID, nil
+	useWebhooks := b.shouldMessageUseWebhooks(&msg)
+	if useWebhooks && msg.Event != config.EventMsgDelete && msg.ParentID == "" {
+		return b.handleEventWebhook(&msg, channelID)
 	}
 
+	return b.handleEventBotUser(&msg, channelID)
+}
+
+// handleEventDirect handles events via the bot user
+func (b *Bdiscord) handleEventBotUser(msg *config.Message, channelID string) (string, error) {
 	b.Log.Debugf("Broadcasting using token (API)")
 
 	// Delete message
@@ -287,19 +282,19 @@ func (b *Bdiscord) Send(msg config.Message) (string, error) {
 
 	// Upload a file if it exists
 	if msg.Extra != nil {
-		for _, rmsg := range helper.HandleExtra(&msg, b.General) {
-			rmsg.Text = helper.ClipMessage(rmsg.Text, MessageLength)
+		for _, rmsg := range helper.HandleExtra(msg, b.General) {
+			rmsg.Text = helper.ClipMessage(rmsg.Text, MessageLength, b.GetString("MessageClipped"))
 			if _, err := b.c.ChannelMessageSend(channelID, rmsg.Username+rmsg.Text); err != nil {
 				b.Log.Errorf("Could not send message %#v: %s", rmsg, err)
 			}
 		}
 		// check if we have files to upload (from slack, telegram or mattermost)
 		if len(msg.Extra["file"]) > 0 {
-			return b.handleUploadFile(&msg, channelID)
+			return b.handleUploadFile(msg, channelID)
 		}
 	}
 
-	msg.Text = helper.ClipMessage(msg.Text, MessageLength)
+	msg.Text = helper.ClipMessage(msg.Text, MessageLength, b.GetString("MessageClipped"))
 	msg.Text = b.replaceUserMentions(msg.Text)
 
 	// Edit message
@@ -308,54 +303,28 @@ func (b *Bdiscord) Send(msg config.Message) (string, error) {
 		return msg.ID, err
 	}
 
+	m := discordgo.MessageSend{
+		Content:         msg.Username + msg.Text,
+		AllowedMentions: b.getAllowedMentions(),
+	}
+
+	if msg.ParentValid() {
+		m.Reference = &discordgo.MessageReference{
+			MessageID: msg.ParentID,
+			ChannelID: channelID,
+			GuildID:   b.guildID,
+		}
+	}
+
 	// Post normal message
-	res, err := b.c.ChannelMessageSend(channelID, msg.Username+msg.Text)
+	res, err := b.c.ChannelMessageSendComplex(channelID, &m)
 	if err != nil {
 		return "", err
 	}
+
 	return res.ID, nil
 }
 
-// useWebhook returns true if we have a webhook defined somewhere
-func (b *Bdiscord) useWebhook() bool {
-	if b.GetString("WebhookURL") != "" {
-		return true
-	}
-
-	b.channelsMutex.RLock()
-	defer b.channelsMutex.RUnlock()
-
-	for _, channel := range b.channelInfoMap {
-		if channel.Options.WebhookURL != "" {
-			return true
-		}
-	}
-	return false
-}
-
-// isWebhookID returns true if the specified id is used in a defined webhook
-func (b *Bdiscord) isWebhookID(id string) bool {
-	if b.GetString("WebhookURL") != "" {
-		wID, _ := b.splitURL(b.GetString("WebhookURL"))
-		if wID == id {
-			return true
-		}
-	}
-
-	b.channelsMutex.RLock()
-	defer b.channelsMutex.RUnlock()
-
-	for _, channel := range b.channelInfoMap {
-		if channel.Options.WebhookURL != "" {
-			wID, _ := b.splitURL(channel.Options.WebhookURL)
-			if wID == id {
-				return true
-			}
-		}
-	}
-	return false
-}
-
 // handleUploadFile handles native upload of files
 func (b *Bdiscord) handleUploadFile(msg *config.Message, channelID string) (string, error) {
 	var err error
@@ -367,8 +336,9 @@ func (b *Bdiscord) handleUploadFile(msg *config.Message, channelID string) (stri
 			Reader:      bytes.NewReader(*fi.Data),
 		}
 		m := discordgo.MessageSend{
-			Content: msg.Username + fi.Comment,
-			Files:   []*discordgo.File{&file},
+			Content:         msg.Username + fi.Comment,
+			Files:           []*discordgo.File{&file},
+			AllowedMentions: b.getAllowedMentions(),
 		}
 		_, err = b.c.ChannelMessageSendComplex(channelID, &m)
 		if err != nil {
@@ -377,83 +347,3 @@ func (b *Bdiscord) handleUploadFile(msg *config.Message, channelID string) (stri
 	}
 	return "", nil
 }
-
-// webhookSend send one or more message via webhook, taking care of file
-// uploads (from slack, telegram or mattermost).
-// Returns messageID and error.
-func (b *Bdiscord) webhookSend(msg *config.Message, webhookID, token string) (*discordgo.Message, error) {
-	var (
-		res *discordgo.Message
-		err error
-	)
-
-	// If avatar is unset, check if UseLocalAvatar contains the message's
-	// account or protocol, and if so, try to find a local avatar
-	if msg.Avatar == "" {
-		for _, val := range b.GetStringSlice("UseLocalAvatar") {
-			if msg.Protocol == val || msg.Account == val {
-				if avatar := b.findAvatar(msg); avatar != "" {
-					msg.Avatar = avatar
-				}
-				break
-			}
-		}
-	}
-
-	// WebhookParams can have either `Content` or `File`.
-
-	// We can't send empty messages.
-	if msg.Text != "" {
-		res, err = b.c.WebhookExecute(
-			webhookID,
-			token,
-			true,
-			&discordgo.WebhookParams{
-				Content:   msg.Text,
-				Username:  msg.Username,
-				AvatarURL: msg.Avatar,
-			},
-		)
-		if err != nil {
-			b.Log.Errorf("Could not send text (%s) for message %#v: %s", msg.Text, msg, err)
-		}
-	}
-
-	if msg.Extra != nil {
-		for _, f := range msg.Extra["file"] {
-			fi := f.(config.FileInfo)
-			file := discordgo.File{
-				Name:        fi.Name,
-				ContentType: "",
-				Reader:      bytes.NewReader(*fi.Data),
-			}
-			content := ""
-			if msg.Text == "" {
-				content = fi.Comment
-			}
-			_, e2 := b.c.WebhookExecute(
-				webhookID,
-				token,
-				false,
-				&discordgo.WebhookParams{
-					Username:  msg.Username,
-					AvatarURL: msg.Avatar,
-					File:      &file,
-					Content:   content,
-				},
-			)
-			if e2 != nil {
-				b.Log.Errorf("Could not send file %#v for message %#v: %s", file, msg, e2)
-			}
-		}
-	}
-	return res, err
-}
-
-func (b *Bdiscord) findAvatar(m *config.Message) string {
-	member, err := b.getGuildMemberByNick(m.Username)
-	if err != nil {
-		return ""
-	}
-	return member.User.AvatarURL("")
-}

bridge/discord/handlers.go

@@ -36,6 +36,11 @@ func (b *Bdiscord) messageTyping(s *discordgo.Session, m *discordgo.TypingStart)
 		return
 	}
 
+	// Ignore our own typing messages
+	if m.UserID == b.userID {
+		return
+	}
+
 	rmsg := config.Message{Account: b.Account, Event: config.EventUserTyping}
 	rmsg.Channel = b.getChannelName(m.ChannelID)
 	b.Remote <- rmsg
@@ -64,7 +69,7 @@ func (b *Bdiscord) messageCreate(s *discordgo.Session, m *discordgo.MessageCreat
 		return
 	}
 	// if using webhooks, do not relay if it's ours
-	if b.useWebhook() && m.Author.Bot && b.isWebhookID(m.Author.ID) {
+	if m.Author.Bot && b.transmitter.HasWebhook(m.Author.ID) {
 		return
 	}
 
@@ -119,6 +124,14 @@ func (b *Bdiscord) messageCreate(s *discordgo.Session, m *discordgo.MessageCreat
 		rmsg.Event = config.EventUserAction
 	}
 
+	// Replace emotes
+	rmsg.Text = replaceEmotes(rmsg.Text)
+
+	// Add our parent id if it exists, and if it's not referring to a message in another channel
+	if ref := m.MessageReference; ref != nil && ref.ChannelID == m.ChannelID {
+		rmsg.ParentID = ref.MessageID
+	}
+
 	b.Log.Debugf("<= Sending message from %s on %s to gateway", m.Author.Username, b.Account)
 	b.Log.Debugf("<= Message is %#v", rmsg)
 	b.Remote <- rmsg
@@ -209,7 +222,7 @@ func handleEmbed(embed *discordgo.MessageEmbed) string {
 
 		i++
 		if i == 1 {
-			result += "embed: " + e
+			result += " embed: " + e
 			continue
 		}
 

bridge/discord/handlers_test.go

@@ -20,14 +20,14 @@ func TestHandleEmbed(t *testing.T) {
 			embed: &discordgo.MessageEmbed{
 				Title: "blah",
 			},
-			result: "embed: blah\n",
+			result: " embed: blah\n",
 		},
 		"two": {
 			embed: &discordgo.MessageEmbed{
 				Title:       "blah",
 				Description: "blah2",
 			},
-			result: "embed: blah - blah2\n",
+			result: " embed: blah - blah2\n",
 		},
 		"three": {
 			embed: &discordgo.MessageEmbed{
@@ -35,20 +35,20 @@ func TestHandleEmbed(t *testing.T) {
 				Description: "blah2",
 				URL:         "blah3",
 			},
-			result: "embed: blah - blah2 - blah3\n",
+			result: " embed: blah - blah2 - blah3\n",
 		},
 		"twob": {
 			embed: &discordgo.MessageEmbed{
 				Description: "blah2",
 				URL:         "blah3",
 			},
-			result: "embed: blah2 - blah3\n",
+			result: " embed: blah2 - blah3\n",
 		},
 		"oneb": {
 			embed: &discordgo.MessageEmbed{
 				URL: "blah3",
 			},
-			result: "embed: blah3\n",
+			result: " embed: blah3\n",
 		},
 	}
 

bridge/discord/helpers.go

@@ -9,6 +9,30 @@ import (
 	"github.com/matterbridge/discordgo"
 )
 
+func (b *Bdiscord) getAllowedMentions() *discordgo.MessageAllowedMentions {
+	// If AllowMention is not specified, then allow all mentions (default Discord behavior)
+	if !b.IsKeySet("AllowMention") {
+		return nil
+	}
+
+	// Otherwise, allow only the mentions that are specified
+	allowedMentionTypes := make([]discordgo.AllowedMentionType, 0, 3)
+	for _, m := range b.GetStringSlice("AllowMention") {
+		switch m {
+		case "everyone":
+			allowedMentionTypes = append(allowedMentionTypes, discordgo.AllowedMentionTypeEveryone)
+		case "roles":
+			allowedMentionTypes = append(allowedMentionTypes, discordgo.AllowedMentionTypeRoles)
+		case "users":
+			allowedMentionTypes = append(allowedMentionTypes, discordgo.AllowedMentionTypeUsers)
+		}
+	}
+
+	return &discordgo.MessageAllowedMentions{
+		Parse: allowedMentionTypes,
+	}
+}
+
 func (b *Bdiscord) getNick(user *discordgo.User, guildID string) string {
 	b.membersMutex.RLock()
 	defer b.membersMutex.RUnlock()
@@ -137,6 +161,7 @@ var (
 	// See https://discordapp.com/developers/docs/reference#message-formatting.
 	channelMentionRE = regexp.MustCompile("<#[0-9]+>")
 	userMentionRE    = regexp.MustCompile("@[^@\n]{1,32}")
+	emoteRE          = regexp.MustCompile(`<a?(:\w+:)\d+>`)
 )
 
 func (b *Bdiscord) replaceChannelMentions(text string) string {
@@ -182,15 +207,20 @@ func (b *Bdiscord) replaceUserMentions(text string) string {
 	return userMentionRE.ReplaceAllStringFunc(text, replaceUserMentionFunc)
 }
 
+func replaceEmotes(text string) string {
+	return emoteRE.ReplaceAllString(text, "$1")
+}
+
 func (b *Bdiscord) replaceAction(text string) (string, bool) {
-	if strings.HasPrefix(text, "_") && strings.HasSuffix(text, "_") {
-		return text[1 : len(text)-1], true
+	length := len(text)
+	if length > 1 && text[0] == '_' && text[length-1] == '_' {
+		return text[1 : length-1], true
 	}
 	return text, false
 }
 
 // splitURL splits a webhookURL and returns the ID and token.
-func (b *Bdiscord) splitURL(url string) (string, string) {
+func (b *Bdiscord) splitURL(url string) (string, string, bool) {
 	const (
 		expectedWebhookSplitCount = 7
 		webhookIdxID              = 5
@@ -198,9 +228,9 @@ func (b *Bdiscord) splitURL(url string) (string, string) {
 	)
 	webhookURLSplit := strings.Split(url, "/")
 	if len(webhookURLSplit) != expectedWebhookSplitCount {
-		b.Log.Fatalf("%s is no correct discord WebhookURL", url)
+		return "", "", false
 	}
-	return webhookURLSplit[webhookIdxID], webhookURLSplit[webhookIdxToken]
+	return webhookURLSplit[webhookIdxID], webhookURLSplit[webhookIdxToken], true
 }
 
 func enumerateUsernames(s string) []string {

bridge/discord/transmitter/transmitter.go

@@ -0,0 +1,257 @@
+// Package transmitter provides functionality for transmitting
+// arbitrary webhook messages to Discord.
+//
+// The package provides the following functionality:
+//
+// - Creating new webhooks, whenever necessary
+// - Loading webhooks that we have previously created
+// - Sending new messages
+// - Editing messages, via message ID
+// - Deleting messages, via message ID
+//
+// The package has been designed for matterbridge, but with other
+// Go bots in mind. The public API should be matterbridge-agnostic.
+package transmitter
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/matterbridge/discordgo"
+	log "github.com/sirupsen/logrus"
+)
+
+// A Transmitter represents a message manager for a single guild.
+type Transmitter struct {
+	session    *discordgo.Session
+	guild      string
+	title      string
+	autoCreate bool
+
+	// channelWebhooks maps from a channel ID to a webhook instance
+	channelWebhooks map[string]*discordgo.Webhook
+
+	mutex sync.RWMutex
+
+	Log *log.Entry
+}
+
+// ErrWebhookNotFound is returned when a valid webhook for this channel/message combination does not exist
+var ErrWebhookNotFound = errors.New("webhook for this channel and message does not exist")
+
+// ErrPermissionDenied is returned if the bot does not have permission to manage webhooks.
+//
+// Bots can be granted a guild-wide permission and channel-specific permissions to manage webhooks.
+// Despite potentially having guild-wide permission, channel specific overrides could deny a bot's permission to manage webhooks.
+var ErrPermissionDenied = errors.New("missing 'Manage Webhooks' permission")
+
+// New returns a new Transmitter given a Discord session, guild ID, and title.
+func New(session *discordgo.Session, guild string, title string, autoCreate bool) *Transmitter {
+	return &Transmitter{
+		session:    session,
+		guild:      guild,
+		title:      title,
+		autoCreate: autoCreate,
+
+		channelWebhooks: make(map[string]*discordgo.Webhook),
+
+		Log: log.NewEntry(log.StandardLogger()),
+	}
+}
+
+// Send transmits a message to the given channel with the provided webhook data, and waits until Discord responds with message data.
+func (t *Transmitter) Send(channelID string, params *discordgo.WebhookParams) (*discordgo.Message, error) {
+	wh, err := t.getOrCreateWebhook(channelID)
+	if err != nil {
+		return nil, err
+	}
+
+	msg, err := t.session.WebhookExecute(wh.ID, wh.Token, true, params)
+	if err != nil {
+		return nil, fmt.Errorf("execute failed: %w", err)
+	}
+
+	return msg, nil
+}
+
+// Edit will edit a message in a channel, if possible.
+func (t *Transmitter) Edit(channelID string, messageID string, params *discordgo.WebhookParams) error {
+	wh := t.getWebhook(channelID)
+
+	if wh == nil {
+		return ErrWebhookNotFound
+	}
+
+	uri := discordgo.EndpointWebhookToken(wh.ID, wh.Token) + "/messages/" + messageID
+	_, err := t.session.RequestWithBucketID("PATCH", uri, params, discordgo.EndpointWebhookToken("", ""))
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// HasWebhook checks whether the transmitter is using a particular webhook.
+func (t *Transmitter) HasWebhook(id string) bool {
+	t.mutex.RLock()
+	defer t.mutex.RUnlock()
+
+	for _, wh := range t.channelWebhooks {
+		if wh.ID == id {
+			return true
+		}
+	}
+
+	return false
+}
+
+// AddWebhook allows you to register a channel's webhook with the transmitter.
+func (t *Transmitter) AddWebhook(channelID string, webhook *discordgo.Webhook) bool {
+	t.Log.Debugf("Manually added webhook %#v to channel %#v", webhook.ID, channelID)
+	t.mutex.Lock()
+	defer t.mutex.Unlock()
+
+	_, replaced := t.channelWebhooks[channelID]
+	t.channelWebhooks[channelID] = webhook
+	return replaced
+}
+
+// RefreshGuildWebhooks loads "relevant" webhooks into the transmitter, with careful permission handling.
+//
+// Notes:
+//
+// - A webhook is "relevant" if it was created by this bot -- the ApplicationID should match the bot's ID.
+// - The term "having permission" means having the "Manage Webhooks" permission. See ErrPermissionDenied for more information.
+// - This function is additive and will not unload previously loaded webhooks.
+// - A nil channelIDs slice is treated the same as an empty one.
+//
+// If the bot has guild-wide permission:
+//
+// 1. it will load any "relevant" webhooks from the entire guild
+// 2. the given slice is ignored
+//
+// If the bot does not have guild-wide permission:
+//
+// 1. it will load any "relevant" webhooks in each channel
+// 2. a single error will be returned if any error occurs (incl. if there is no permission for any of these channels)
+//
+// If any channel has more than one "relevant" webhook, it will randomly pick one.
+func (t *Transmitter) RefreshGuildWebhooks(channelIDs []string) error {
+	t.Log.Debugln("Refreshing guild webhooks")
+
+	botID, err := getDiscordUserID(t.session)
+	if err != nil {
+		return fmt.Errorf("could not get current user: %w", err)
+	}
+
+	// Get all existing webhooks
+	hooks, err := t.session.GuildWebhooks(t.guild)
+	if err != nil {
+		switch {
+		case isDiscordPermissionError(err):
+			// We fallback on manually fetching hooks from individual channels
+			// if we don't have the "Manage Webhooks" permission globally.
+			// We can only do this if we were provided channelIDs, though.
+			if len(channelIDs) == 0 {
+				return ErrPermissionDenied
+			}
+			t.Log.Debugln("Missing global 'Manage Webhooks' permission, falling back on per-channel permission")
+			return t.fetchChannelsHooks(channelIDs, botID)
+		default:
+			return fmt.Errorf("could not get webhooks: %w", err)
+		}
+	}
+
+	t.Log.Debugln("Refreshing guild webhooks using global permission")
+	t.assignHooksByAppID(hooks, botID, false)
+	return nil
+}
+
+// createWebhook creates a webhook for a specific channel.
+func (t *Transmitter) createWebhook(channel string) (*discordgo.Webhook, error) {
+	t.mutex.Lock()
+	defer t.mutex.Unlock()
+
+	wh, err := t.session.WebhookCreate(channel, t.title+time.Now().Format(" 3:04:05PM"), "")
+	if err != nil {
+		return nil, err
+	}
+
+	t.channelWebhooks[channel] = wh
+	return wh, nil
+}
+
+func (t *Transmitter) getWebhook(channel string) *discordgo.Webhook {
+	t.mutex.RLock()
+	defer t.mutex.RUnlock()
+
+	return t.channelWebhooks[channel]
+}
+
+func (t *Transmitter) getOrCreateWebhook(channelID string) (*discordgo.Webhook, error) {
+	// If we have a webhook for this channel, immediately return it
+	wh := t.getWebhook(channelID)
+	if wh != nil {
+		return wh, nil
+	}
+
+	// Early exit if we don't want to automatically create one
+	if !t.autoCreate {
+		return nil, ErrWebhookNotFound
+	}
+
+	t.Log.Infof("Creating a webhook for %s\n", channelID)
+	wh, err := t.createWebhook(channelID)
+	if err != nil {
+		return nil, fmt.Errorf("could not create webhook: %w", err)
+	}
+
+	return wh, nil
+}
+
+// fetchChannelsHooks fetches hooks for the given channelIDs and calls assignHooksByAppID for each channel's hooks
+func (t *Transmitter) fetchChannelsHooks(channelIDs []string, botID string) error {
+	// For each channel, search for relevant hooks
+	var failedHooks []string
+	for _, channelID := range channelIDs {
+		hooks, err := t.session.ChannelWebhooks(channelID)
+		if err != nil {
+			failedHooks = append(failedHooks, "\n- "+channelID+": "+err.Error())
+			continue
+		}
+		t.assignHooksByAppID(hooks, botID, true)
+	}
+
+	// Compose an error if any hooks failed
+	if len(failedHooks) > 0 {
+		return errors.New("failed to fetch hooks:" + strings.Join(failedHooks, ""))
+	}
+
+	return nil
+}
+
+func (t *Transmitter) assignHooksByAppID(hooks []*discordgo.Webhook, appID string, channelTargeted bool) {
+	logLine := "Picking up webhook"
+	if channelTargeted {
+		logLine += " (channel targeted)"
+	}
+
+	t.mutex.Lock()
+	defer t.mutex.Unlock()
+
+	for _, wh := range hooks {
+		if wh.ApplicationID != appID {
+			continue
+		}
+
+		t.channelWebhooks[wh.ChannelID] = wh
+		t.Log.WithFields(log.Fields{
+			"id":      wh.ID,
+			"name":    wh.Name,
+			"channel": wh.ChannelID,
+		}).Println(logLine)
+	}
+}

bridge/discord/transmitter/utils.go

@@ -0,0 +1,32 @@
+package transmitter
+
+import (
+	"github.com/matterbridge/discordgo"
+)
+
+// isDiscordPermissionError returns false for nil, and true if a Discord RESTError with code discordgo.ErrorCodeMissionPermissions
+func isDiscordPermissionError(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	restErr, ok := err.(*discordgo.RESTError)
+	if !ok {
+		return false
+	}
+
+	return restErr.Message != nil && restErr.Message.Code == discordgo.ErrCodeMissingPermissions
+}
+
+// getDiscordUserID gets own user ID from state, and fallback on API request
+func getDiscordUserID(session *discordgo.Session) (string, error) {
+	if user := session.State.User; user != nil {
+		return user.ID, nil
+	}
+
+	user, err := session.User("@me")
+	if err != nil {
+		return "", err
+	}
+	return user.ID, nil
+}

bridge/discord/webhook.go

@@ -0,0 +1,150 @@
+package bdiscord
+
+import (
+	"bytes"
+
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/42wim/matterbridge/bridge/helper"
+	"github.com/matterbridge/discordgo"
+)
+
+// shouldMessageUseWebhooks checks if have a channel specific webhook, if we're not using auto webhooks
+func (b *Bdiscord) shouldMessageUseWebhooks(msg *config.Message) bool {
+	if b.useAutoWebhooks {
+		return true
+	}
+
+	b.channelsMutex.RLock()
+	defer b.channelsMutex.RUnlock()
+	if ci, ok := b.channelInfoMap[msg.Channel+b.Account]; ok {
+		if ci.Options.WebhookURL != "" {
+			return true
+		}
+	}
+	return false
+}
+
+// maybeGetLocalAvatar checks if UseLocalAvatar contains the message's
+// account or protocol, and if so, returns the Discord avatar (if exists)
+func (b *Bdiscord) maybeGetLocalAvatar(msg *config.Message) string {
+	for _, val := range b.GetStringSlice("UseLocalAvatar") {
+		if msg.Protocol != val && msg.Account != val {
+			continue
+		}
+
+		member, err := b.getGuildMemberByNick(msg.Username)
+		if err != nil {
+			return ""
+		}
+
+		return member.User.AvatarURL("")
+	}
+	return ""
+}
+
+// webhookSend send one or more message via webhook, taking care of file
+// uploads (from slack, telegram or mattermost).
+// Returns messageID and error.
+func (b *Bdiscord) webhookSend(msg *config.Message, channelID string) (*discordgo.Message, error) {
+	var (
+		res *discordgo.Message
+		err error
+	)
+
+	// If avatar is unset, mutate the message to include the local avatar (but only if settings say we should do this)
+	if msg.Avatar == "" {
+		msg.Avatar = b.maybeGetLocalAvatar(msg)
+	}
+
+	// WebhookParams can have either `Content` or `File`.
+
+	// We can't send empty messages.
+	if msg.Text != "" {
+		res, err = b.transmitter.Send(
+			channelID,
+			&discordgo.WebhookParams{
+				Content:         msg.Text,
+				Username:        msg.Username,
+				AvatarURL:       msg.Avatar,
+				AllowedMentions: b.getAllowedMentions(),
+			},
+		)
+		if err != nil {
+			b.Log.Errorf("Could not send text (%s) for message %#v: %s", msg.Text, msg, err)
+		}
+	}
+
+	if msg.Extra != nil {
+		for _, f := range msg.Extra["file"] {
+			fi := f.(config.FileInfo)
+			file := discordgo.File{
+				Name:        fi.Name,
+				ContentType: "",
+				Reader:      bytes.NewReader(*fi.Data),
+			}
+			content := ""
+			if msg.Text == "" {
+				content = fi.Comment
+			}
+			_, e2 := b.transmitter.Send(
+				channelID,
+				&discordgo.WebhookParams{
+					Username:        msg.Username,
+					AvatarURL:       msg.Avatar,
+					File:            &file,
+					Content:         content,
+					AllowedMentions: b.getAllowedMentions(),
+				},
+			)
+			if e2 != nil {
+				b.Log.Errorf("Could not send file %#v for message %#v: %s", file, msg, e2)
+			}
+		}
+	}
+	return res, err
+}
+
+func (b *Bdiscord) handleEventWebhook(msg *config.Message, channelID string) (string, error) {
+	// skip events
+	if msg.Event != "" && msg.Event != config.EventUserAction && msg.Event != config.EventJoinLeave && msg.Event != config.EventTopicChange {
+		return "", nil
+	}
+
+	// skip empty messages
+	if msg.Text == "" && (msg.Extra == nil || len(msg.Extra["file"]) == 0) {
+		b.Log.Debugf("Skipping empty message %#v", msg)
+		return "", nil
+	}
+
+	msg.Text = helper.ClipMessage(msg.Text, MessageLength, b.GetString("MessageClipped"))
+	msg.Text = b.replaceUserMentions(msg.Text)
+	// discord username must be [0..32] max
+	if len(msg.Username) > 32 {
+		msg.Username = msg.Username[0:32]
+	}
+
+	if msg.ID != "" {
+		b.Log.Debugf("Editing webhook message")
+		err := b.transmitter.Edit(channelID, msg.ID, &discordgo.WebhookParams{
+			Content:         msg.Text,
+			Username:        msg.Username,
+			AllowedMentions: b.getAllowedMentions(),
+		})
+		if err == nil {
+			return msg.ID, nil
+		}
+		b.Log.Errorf("Could not edit webhook message: %s", err)
+	}
+
+	b.Log.Debugf("Processing webhook sending for message %#v", msg)
+	discordMsg, err := b.webhookSend(msg, channelID)
+	if err != nil {
+		b.Log.Errorf("Could not broadcast via webhook for message %#v: %s", msg, err)
+		return "", err
+	}
+	if discordMsg == nil {
+		return "", nil
+	}
+
+	return discordMsg.ID, nil
+}

bridge/helper/helper.go

@@ -5,7 +5,10 @@ import (
 	"fmt"
 	"image/png"
 	"io"
+	"io/ioutil"
 	"net/http"
+	"os"
+	"os/exec"
 	"regexp"
 	"strings"
 	"time"
@@ -48,6 +51,30 @@ func DownloadFileAuth(url string, auth string) (*[]byte, error) {
 	return &data, nil
 }
 
+// DownloadFileAuthRocket downloads the given URL using the specified Rocket user ID and authentication token.
+func DownloadFileAuthRocket(url, token, userID string) (*[]byte, error) {
+	var buf bytes.Buffer
+	client := &http.Client{
+		Timeout: time.Second * 5,
+	}
+	req, err := http.NewRequest("GET", url, nil)
+
+	req.Header.Add("X-Auth-Token", token)
+	req.Header.Add("X-User-Id", userID)
+
+	if err != nil {
+		return nil, err
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	_, err = io.Copy(&buf, resp.Body)
+	data := buf.Bytes()
+	return &data, err
+}
+
 // GetSubLines splits messages in newline-delimited lines. If maxLineLength is
 // specified as non-zero GetSubLines will also clip long lines to the maximum
 // length and insert a warning marker that the line was clipped.
@@ -55,8 +82,10 @@ func DownloadFileAuth(url string, auth string) (*[]byte, error) {
 // TODO: The current implementation has the inconvenient that it disregards
 // word boundaries when splitting but this is hard to solve without potentially
 // breaking formatting and other stylistic effects.
-func GetSubLines(message string, maxLineLength int) []string {
-	const clippingMessage = " <clipped message>"
+func GetSubLines(message string, maxLineLength int, clippingMessage string) []string {
+	if clippingMessage == "" {
+		clippingMessage = " <clipped message>"
+	}
 
 	var lines []string
 	for _, line := range strings.Split(strings.TrimSpace(message), "\n") {
@@ -166,8 +195,11 @@ func RemoveEmptyNewLines(msg string) string {
 
 // ClipMessage trims a message to the specified length if it exceeds it and adds a warning
 // to the message in case it does so.
-func ClipMessage(text string, length int) string {
-	const clippingMessage = " <clipped message>"
+func ClipMessage(text string, length int, clippingMessage string) string {
+	if clippingMessage == "" {
+		clippingMessage = " <clipped message>"
+	}
+
 	if len(text) > length {
 		text = text[:length-len(clippingMessage)]
 		if r, size := utf8.DecodeLastRuneInString(text); r == utf8.RuneError {
@@ -180,7 +212,7 @@ func ClipMessage(text string, length int) string {
 
 // ParseMarkdown takes in an input string as markdown and parses it to html
 func ParseMarkdown(input string) string {
-	extensions := parser.HardLineBreak | parser.NoIntraEmphasis
+	extensions := parser.HardLineBreak | parser.NoIntraEmphasis | parser.FencedCode
 	markdownParser := parser.NewWithExtensions(extensions)
 	renderer := html.NewRenderer(html.RendererOptions{
 		Flags: 0,
@@ -192,7 +224,7 @@ func ParseMarkdown(input string) string {
 	return res
 }
 
-// ConvertWebPToPNG convert input data (which should be WebP format to PNG format)
+// ConvertWebPToPNG converts input data (which should be WebP format) to PNG format
 func ConvertWebPToPNG(data *[]byte) error {
 	r := bytes.NewReader(*data)
 	m, err := webp.Decode(r)
@@ -207,3 +239,66 @@ func ConvertWebPToPNG(data *[]byte) error {
 	*data = w.Bytes()
 	return nil
 }
+
+// CanConvertTgsToX Checks whether the external command necessary for ConvertTgsToX works.
+func CanConvertTgsToX() error {
+	// We depend on the fact that `lottie_convert.py --help` has exit status 0.
+	// Hyrum's Law predicted this, and Murphy's Law predicts that this will break eventually.
+	// However, there is no alternative like `lottie_convert.py --is-properly-installed`
+	cmd := exec.Command("lottie_convert.py", "--help")
+	return cmd.Run()
+}
+
+// ConvertTgsToWebP convert input data (which should be tgs format) to WebP format
+// This relies on an external command, which is ugly, but works.
+func ConvertTgsToX(data *[]byte, outputFormat string, logger *logrus.Entry) error {
+	// lottie can't handle input from a pipe, so write to a temporary file:
+	tmpInFile, err := ioutil.TempFile(os.TempDir(), "matterbridge-lottie-input-*.tgs")
+	if err != nil {
+		return err
+	}
+	tmpInFileName := tmpInFile.Name()
+	defer func() {
+		if removeErr := os.Remove(tmpInFileName); removeErr != nil {
+			logger.Errorf("Could not delete temporary (input) file %s: %v", tmpInFileName, removeErr)
+		}
+	}()
+	// lottie can handle writing to a pipe, but there is no way to do that platform-independently.
+	// "/dev/stdout" won't work on Windows, and "-" upsets Cairo for some reason. So we need another file:
+	tmpOutFile, err := ioutil.TempFile(os.TempDir(), "matterbridge-lottie-output-*.data")
+	if err != nil {
+		return err
+	}
+	tmpOutFileName := tmpOutFile.Name()
+	defer func() {
+		if removeErr := os.Remove(tmpOutFileName); removeErr != nil {
+			logger.Errorf("Could not delete temporary (output) file %s: %v", tmpOutFileName, removeErr)
+		}
+	}()
+
+	if _, writeErr := tmpInFile.Write(*data); writeErr != nil {
+		return writeErr
+	}
+	// Must close before calling lottie to avoid data races:
+	if closeErr := tmpInFile.Close(); closeErr != nil {
+		return closeErr
+	}
+
+	// Call lottie to transform:
+	cmd := exec.Command("lottie_convert.py", "--input-format", "lottie", "--output-format", outputFormat, tmpInFileName, tmpOutFileName)
+	cmd.Stdout = nil
+	cmd.Stderr = nil
+	// NB: lottie writes progress into to stderr in all cases.
+	_, stderr := cmd.Output()
+	if stderr != nil {
+		// 'stderr' already contains some parts of Stderr, because it was set to 'nil'.
+		return stderr
+	}
+	dataContents, err := ioutil.ReadFile(tmpOutFileName)
+	if err != nil {
+		return err
+	}
+
+	*data = dataContents
+	return nil
+}

bridge/helper/helper_test.go

@@ -10,98 +10,96 @@ import (
 
 const testLineLength = 64
 
-var (
-	lineSplittingTestCases = map[string]struct {
-		input          string
-		splitOutput    []string
-		nonSplitOutput []string
-	}{
-		"Short single-line message": {
-			input:          "short",
-			splitOutput:    []string{"short"},
-			nonSplitOutput: []string{"short"},
+var lineSplittingTestCases = map[string]struct {
+	input          string
+	splitOutput    []string
+	nonSplitOutput []string
+}{
+	"Short single-line message": {
+		input:          "short",
+		splitOutput:    []string{"short"},
+		nonSplitOutput: []string{"short"},
+	},
+	"Long single-line message": {
+		input: "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.",
+		splitOutput: []string{
+			"Lorem ipsum dolor sit amet, consectetur adipis <clipped message>",
+			"cing elit, sed do eiusmod tempor incididunt ut <clipped message>",
+			" labore et dolore magna aliqua.",
 		},
-		"Long single-line message": {
-			input: "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.",
-			splitOutput: []string{
-				"Lorem ipsum dolor sit amet, consectetur adipis <clipped message>",
-				"cing elit, sed do eiusmod tempor incididunt ut <clipped message>",
-				" labore et dolore magna aliqua.",
-			},
-			nonSplitOutput: []string{"Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua."},
+		nonSplitOutput: []string{"Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua."},
+	},
+	"Short multi-line message": {
+		input: "I\ncan't\nget\nno\nsatisfaction!",
+		splitOutput: []string{
+			"I",
+			"can't",
+			"get",
+			"no",
+			"satisfaction!",
 		},
-		"Short multi-line message": {
-			input: "I\ncan't\nget\nno\nsatisfaction!",
-			splitOutput: []string{
-				"I",
-				"can't",
-				"get",
-				"no",
-				"satisfaction!",
-			},
-			nonSplitOutput: []string{
-				"I",
-				"can't",
-				"get",
-				"no",
-				"satisfaction!",
-			},
+		nonSplitOutput: []string{
+			"I",
+			"can't",
+			"get",
+			"no",
+			"satisfaction!",
 		},
-		"Long multi-line message": {
-			input: "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.\n" +
-				"Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.\n" +
-				"Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.\n" +
-				"Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.",
-			splitOutput: []string{
-				"Lorem ipsum dolor sit amet, consectetur adipis <clipped message>",
-				"cing elit, sed do eiusmod tempor incididunt ut <clipped message>",
-				" labore et dolore magna aliqua.",
-				"Ut enim ad minim veniam, quis nostrud exercita <clipped message>",
-				"tion ullamco laboris nisi ut aliquip ex ea com <clipped message>",
-				"modo consequat.",
-				"Duis aute irure dolor in reprehenderit in volu <clipped message>",
-				"ptate velit esse cillum dolore eu fugiat nulla <clipped message>",
-				" pariatur.",
-				"Excepteur sint occaecat cupidatat non proident <clipped message>",
-				", sunt in culpa qui officia deserunt mollit an <clipped message>",
-				"im id est laborum.",
-			},
-			nonSplitOutput: []string{
-				"Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.",
-				"Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.",
-				"Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.",
-				"Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.",
-			},
+	},
+	"Long multi-line message": {
+		input: "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.\n" +
+			"Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.\n" +
+			"Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.\n" +
+			"Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.",
+		splitOutput: []string{
+			"Lorem ipsum dolor sit amet, consectetur adipis <clipped message>",
+			"cing elit, sed do eiusmod tempor incididunt ut <clipped message>",
+			" labore et dolore magna aliqua.",
+			"Ut enim ad minim veniam, quis nostrud exercita <clipped message>",
+			"tion ullamco laboris nisi ut aliquip ex ea com <clipped message>",
+			"modo consequat.",
+			"Duis aute irure dolor in reprehenderit in volu <clipped message>",
+			"ptate velit esse cillum dolore eu fugiat nulla <clipped message>",
+			" pariatur.",
+			"Excepteur sint occaecat cupidatat non proident <clipped message>",
+			", sunt in culpa qui officia deserunt mollit an <clipped message>",
+			"im id est laborum.",
 		},
-		"Message ending with new-line.": {
-			input:          "Newline ending\n",
-			splitOutput:    []string{"Newline ending"},
-			nonSplitOutput: []string{"Newline ending"},
+		nonSplitOutput: []string{
+			"Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.",
+			"Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.",
+			"Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.",
+			"Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.",
 		},
-		"Long message containing UTF-8 multi-byte runes": {
-			input: "不布人個我此而及單石業喜資富下我河下日沒一我臺空達的常景便物沒為……子大我別名解成？生賣的全直黑，我自我結毛分洲了世當，是政福那是東；斯說",
-			splitOutput: []string{
-				"不布人個我此而及單石業喜資富下 <clipped message>",
-				"我河下日沒一我臺空達的常景便物 <clipped message>",
-				"沒為……子大我別名解成？生賣的 <clipped message>",
-				"全直黑，我自我結毛分洲了世當， <clipped message>",
-				"是政福那是東；斯說",
-			},
-			nonSplitOutput: []string{"不布人個我此而及單石業喜資富下我河下日沒一我臺空達的常景便物沒為……子大我別名解成？生賣的全直黑，我自我結毛分洲了世當，是政福那是東；斯說"},
+	},
+	"Message ending with new-line.": {
+		input:          "Newline ending\n",
+		splitOutput:    []string{"Newline ending"},
+		nonSplitOutput: []string{"Newline ending"},
+	},
+	"Long message containing UTF-8 multi-byte runes": {
+		input: "不布人個我此而及單石業喜資富下我河下日沒一我臺空達的常景便物沒為……子大我別名解成？生賣的全直黑，我自我結毛分洲了世當，是政福那是東；斯說",
+		splitOutput: []string{
+			"不布人個我此而及單石業喜資富下 <clipped message>",
+			"我河下日沒一我臺空達的常景便物 <clipped message>",
+			"沒為……子大我別名解成？生賣的 <clipped message>",
+			"全直黑，我自我結毛分洲了世當， <clipped message>",
+			"是政福那是東；斯說",
 		},
-	}
-)
+		nonSplitOutput: []string{"不布人個我此而及單石業喜資富下我河下日沒一我臺空達的常景便物沒為……子大我別名解成？生賣的全直黑，我自我結毛分洲了世當，是政福那是東；斯說"},
+	},
+}
 
 func TestGetSubLines(t *testing.T) {
 	for testname, testcase := range lineSplittingTestCases {
-		splitLines := GetSubLines(testcase.input, testLineLength)
+		splitLines := GetSubLines(testcase.input, testLineLength, "")
 		assert.Equalf(t, testcase.splitOutput, splitLines, "'%s' testcase should give expected lines with splitting.", testname)
 		for _, splitLine := range splitLines {
 			byteLength := len([]byte(splitLine))
 			assert.True(t, byteLength <= testLineLength, "Splitted line '%s' of testcase '%s' should not exceed the maximum byte-length (%d vs. %d).", splitLine, testcase, byteLength, testLineLength)
 		}
 
-		nonSplitLines := GetSubLines(testcase.input, 0)
+		nonSplitLines := GetSubLines(testcase.input, 0, "")
 		assert.Equalf(t, testcase.nonSplitOutput, nonSplitLines, "'%s' testcase should give expected lines without splitting.", testname)
 	}
 }
@@ -110,16 +108,19 @@ func TestConvertWebPToPNG(t *testing.T) {
 	if os.Getenv("LOCAL_TEST") == "" {
 		t.Skip()
 	}
+
 	input, err := ioutil.ReadFile("test.webp")
 	if err != nil {
 		t.Fail()
 	}
+
 	d := &input
 	err = ConvertWebPToPNG(d)
 	if err != nil {
 		t.Fail()
 	}
-	err = ioutil.WriteFile("test.png", *d, 0644)
+
+	err = ioutil.WriteFile("test.png", *d, 0o644) // nolint:gosec
 	if err != nil {
 		t.Fail()
 	}

bridge/irc/handlers.go

@@ -10,8 +10,8 @@ import (
 
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/42wim/matterbridge/bridge/helper"
-	"github.com/dfordsoft/golib/ic"
 	"github.com/lrstanley/girc"
+	"github.com/missdeer/golib/ic"
 	"github.com/paulrosania/go-charset/charset"
 	"github.com/saintfish/chardet"
 
@@ -54,12 +54,12 @@ func (b *Birc) handleFiles(msg *config.Message) bool {
 	for _, f := range msg.Extra["file"] {
 		fi := f.(config.FileInfo)
 		if fi.Comment != "" {
-			msg.Text += fi.Comment + ": "
+			msg.Text += fi.Comment + " : "
 		}
 		if fi.URL != "" {
 			msg.Text = fi.URL
 			if fi.Comment != "" {
-				msg.Text = fi.Comment + ": " + fi.URL
+				msg.Text = fi.Comment + " : " + fi.URL
 			}
 		}
 		b.Local <- config.Message{Text: msg.Text, Username: msg.Username, Channel: msg.Channel, Event: msg.Event}
@@ -67,6 +67,20 @@ func (b *Birc) handleFiles(msg *config.Message) bool {
 	return true
 }
 
+func (b *Birc) handleInvite(client *girc.Client, event girc.Event) {
+	if len(event.Params) != 2 {
+		return
+	}
+
+	channel := event.Params[1]
+
+	b.Log.Debugf("got invite for %s", channel)
+
+	if _, ok := b.channels[channel]; ok {
+		b.i.Cmd.Join(channel)
+	}
+}
+
 func (b *Birc) handleJoinPart(client *girc.Client, event girc.Event) {
 	if len(event.Params) == 0 {
 		b.Log.Debugf("handleJoinPart: empty Params? %#v", event)
@@ -109,14 +123,15 @@ func (b *Birc) handleNewConnection(client *girc.Client, event girc.Event) {
 	i := b.i
 	b.Nick = event.Params[0]
 
-	i.Handlers.Add("PRIVMSG", b.handlePrivMsg)
-	i.Handlers.Add("CTCP_ACTION", b.handlePrivMsg)
+	i.Handlers.AddBg("PRIVMSG", b.handlePrivMsg)
+	i.Handlers.AddBg("CTCP_ACTION", b.handlePrivMsg)
 	i.Handlers.Add(girc.RPL_TOPICWHOTIME, b.handleTopicWhoTime)
-	i.Handlers.Add(girc.NOTICE, b.handleNotice)
-	i.Handlers.Add("JOIN", b.handleJoinPart)
-	i.Handlers.Add("PART", b.handleJoinPart)
-	i.Handlers.Add("QUIT", b.handleJoinPart)
-	i.Handlers.Add("KICK", b.handleJoinPart)
+	i.Handlers.AddBg(girc.NOTICE, b.handleNotice)
+	i.Handlers.AddBg("JOIN", b.handleJoinPart)
+	i.Handlers.AddBg("PART", b.handleJoinPart)
+	i.Handlers.AddBg("QUIT", b.handleJoinPart)
+	i.Handlers.AddBg("KICK", b.handleJoinPart)
+	i.Handlers.Add("INVITE", b.handleInvite)
 }
 
 func (b *Birc) handleNickServ() {
@@ -170,7 +185,14 @@ func (b *Birc) handlePrivMsg(client *girc.Client, event girc.Event) {
 	if b.skipPrivMsg(event) {
 		return
 	}
-	rmsg := config.Message{Username: event.Source.Name, Channel: strings.ToLower(event.Params[0]), Account: b.Account, UserID: event.Source.Ident + "@" + event.Source.Host}
+
+	rmsg := config.Message{
+		Username: event.Source.Name,
+		Channel:  strings.ToLower(event.Params[0]),
+		Account:  b.Account,
+		UserID:   event.Source.Ident + "@" + event.Source.Host,
+	}
+
 	b.Log.Debugf("== Receiving PRIVMSG: %s %s %#v", event.Source.Name, event.Last(), event)
 
 	// set action event
@@ -178,6 +200,11 @@ func (b *Birc) handlePrivMsg(client *girc.Client, event girc.Event) {
 		rmsg.Event = config.EventUserAction
 	}
 
+	// set NOTICE event
+	if event.Command == "NOTICE" {
+		rmsg.Event = config.EventNoticeIRC
+	}
+
 	// strip action, we made an event if it was an action
 	rmsg.Text += event.StripAction()
 

bridge/irc/irc.go

@@ -4,6 +4,7 @@ import (
 	"crypto/tls"
 	"fmt"
 	"hash/crc32"
+	"io/ioutil"
 	"net"
 	"sort"
 	"strconv"
@@ -14,6 +15,7 @@ import (
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/42wim/matterbridge/bridge/helper"
 	"github.com/lrstanley/girc"
+	stripmd "github.com/writeas/go-strip-markdown"
 
 	// We need to import the 'data' package as an implicit dependency.
 	// See: https://godoc.org/github.com/paulrosania/go-charset/charset
@@ -28,6 +30,7 @@ type Birc struct {
 	Local                                     chan config.Message // local queue for flood control
 	FirstConnection, authDone                 bool
 	MessageDelay, MessageQueue, MessageLength int
+	channels                                  map[string]bool
 
 	*bridge.Config
 }
@@ -38,6 +41,8 @@ func New(cfg *bridge.Config) bridge.Bridger {
 	b.Nick = b.GetString("Nick")
 	b.names = make(map[string][]string)
 	b.connected = make(chan error)
+	b.channels = make(map[string]bool)
+
 	if b.GetInt("MessageDelay") == 0 {
 		b.MessageDelay = 1300
 	} else {
@@ -110,6 +115,7 @@ func (b *Birc) Disconnect() error {
 }
 
 func (b *Birc) JoinChannel(channel config.ChannelInfo) error {
+	b.channels[channel.Name] = true
 	// need to check if we have nickserv auth done before joining channels
 	for {
 		if b.authDone {
@@ -156,10 +162,14 @@ func (b *Birc) Send(msg config.Message) (string, error) {
 	}
 
 	var msgLines []string
+	if b.GetBool("StripMarkdown") {
+		msg.Text = stripmd.Strip(msg.Text)
+	}
+
 	if b.GetBool("MessageSplit") {
-		msgLines = helper.GetSubLines(msg.Text, b.MessageLength)
+		msgLines = helper.GetSubLines(msg.Text, b.MessageLength, b.GetString("MessageClipped"))
 	} else {
-		msgLines = helper.GetSubLines(msg.Text, 0)
+		msgLines = helper.GetSubLines(msg.Text, 0, b.GetString("MessageClipped"))
 	}
 	for i := range msgLines {
 		if len(b.Local) >= b.MessageQueue {
@@ -195,22 +205,58 @@ func (b *Birc) doConnect() {
 	}
 }
 
+// Sanitize nicks for RELAYMSG: replace IRC characters with special meanings with "-"
+func sanitizeNick(nick string) string {
+	sanitize := func(r rune) rune {
+		if strings.ContainsRune("!+%@&#$:'\"?*,. ", r) {
+			return '-'
+		}
+		return r
+	}
+	return strings.Map(sanitize, nick)
+}
+
 func (b *Birc) doSend() {
 	rate := time.Millisecond * time.Duration(b.MessageDelay)
 	throttle := time.NewTicker(rate)
 	for msg := range b.Local {
 		<-throttle.C
 		username := msg.Username
-		if b.GetBool("Colornicks") {
-			checksum := crc32.ChecksumIEEE([]byte(msg.Username))
-			colorCode := checksum%14 + 2 // quick fix - prevent white or black color codes
-			username = fmt.Sprintf("\x03%02d%s\x0F", colorCode, msg.Username)
-		}
-		if msg.Event == config.EventUserAction {
-			b.i.Cmd.Action(msg.Channel, username+msg.Text)
+		// Optional support for the proposed RELAYMSG extension, described at
+		// https://github.com/jlu5/ircv3-specifications/blob/master/extensions/relaymsg.md
+		// nolint:nestif
+		if (b.i.HasCapability("overdrivenetworks.com/relaymsg") || b.i.HasCapability("draft/relaymsg")) &&
+			b.GetBool("UseRelayMsg") {
+			username = sanitizeNick(username)
+			text := msg.Text
+
+			// Work around girc chomping leading commas on single word messages?
+			if strings.HasPrefix(text, ":") && !strings.ContainsRune(text, ' ') {
+				text = ":" + text
+			}
+
+			if msg.Event == config.EventUserAction {
+				b.i.Cmd.SendRawf("RELAYMSG %s %s :\x01ACTION %s\x01", msg.Channel, username, text) //nolint:errcheck
+			} else {
+				b.Log.Debugf("Sending RELAYMSG to channel %s: nick=%s", msg.Channel, username)
+				b.i.Cmd.SendRawf("RELAYMSG %s %s :%s", msg.Channel, username, text) //nolint:errcheck
+			}
 		} else {
-			b.Log.Debugf("Sending to channel %s", msg.Channel)
-			b.i.Cmd.Message(msg.Channel, username+msg.Text)
+			if b.GetBool("Colornicks") {
+				checksum := crc32.ChecksumIEEE([]byte(msg.Username))
+				colorCode := checksum%14 + 2 // quick fix - prevent white or black color codes
+				username = fmt.Sprintf("\x03%02d%s\x0F", colorCode, msg.Username)
+			}
+			switch msg.Event {
+			case config.EventUserAction:
+				b.i.Cmd.Action(msg.Channel, username+msg.Text)
+			case config.EventNoticeIRC:
+				b.Log.Debugf("Sending notice to channel %s", msg.Channel)
+				b.i.Cmd.Notice(msg.Channel, username+msg.Text)
+			default:
+				b.Log.Debugf("Sending to channel %s", msg.Channel)
+				b.i.Cmd.Message(msg.Channel, username+msg.Text)
+			}
 		}
 	}
 }
@@ -235,6 +281,18 @@ func (b *Birc) getClient() (*girc.Client, error) {
 		user = user[1:]
 	}
 
+	debug := ioutil.Discard
+	if b.GetInt("DebugLevel") == 2 {
+		debug = b.Log.Writer()
+	}
+
+	pingDelay, err := time.ParseDuration(b.GetString("pingdelay"))
+	if err != nil || pingDelay == 0 {
+		pingDelay = time.Minute
+	}
+
+	b.Log.Debugf("setting pingdelay to %s", pingDelay)
+
 	i := girc.New(girc.Config{
 		Server:     server,
 		ServerPass: b.GetString("Password"),
@@ -244,7 +302,11 @@ func (b *Birc) getClient() (*girc.Client, error) {
 		Name:       b.GetString("Nick"),
 		SSL:        b.GetBool("UseTLS"),
 		TLSConfig:  &tls.Config{InsecureSkipVerify: b.GetBool("SkipTLSVerify"), ServerName: server}, //nolint:gosec
-		PingDelay:  time.Minute,
+		PingDelay:  pingDelay,
+		// skip gIRC internal rate limiting, since we have our own throttling
+		AllowFlood:    true,
+		Debug:         debug,
+		SupportedCaps: map[string][]string{"overdrivenetworks.com/relaymsg": nil, "draft/relaymsg": nil},
 	})
 	return i, nil
 }
@@ -254,12 +316,16 @@ func (b *Birc) endNames(client *girc.Client, event girc.Event) {
 	sort.Strings(b.names[channel])
 	maxNamesPerPost := (300 / b.nicksPerRow()) * b.nicksPerRow()
 	for len(b.names[channel]) > maxNamesPerPost {
-		b.Remote <- config.Message{Username: b.Nick, Text: b.formatnicks(b.names[channel][0:maxNamesPerPost]),
-			Channel: channel, Account: b.Account}
+		b.Remote <- config.Message{
+			Username: b.Nick, Text: b.formatnicks(b.names[channel][0:maxNamesPerPost]),
+			Channel: channel, Account: b.Account,
+		}
 		b.names[channel] = b.names[channel][maxNamesPerPost:]
 	}
-	b.Remote <- config.Message{Username: b.Nick, Text: b.formatnicks(b.names[channel]),
-		Channel: channel, Account: b.Account}
+	b.Remote <- config.Message{
+		Username: b.Nick, Text: b.formatnicks(b.names[channel]),
+		Channel: channel, Account: b.Account,
+	}
 	b.names[channel] = nil
 	b.i.Handlers.Clear(girc.RPL_NAMREPLY)
 	b.i.Handlers.Clear(girc.RPL_ENDOFNAMES)
@@ -270,7 +336,7 @@ func (b *Birc) skipPrivMsg(event girc.Event) bool {
 	b.Nick = b.i.GetNick()
 
 	// freenode doesn't send 001 as first reply
-	if event.Command == "NOTICE" {
+	if event.Command == "NOTICE" && len(event.Params) != 2 {
 		return true
 	}
 	// don't forward queries to the bot
@@ -281,6 +347,15 @@ func (b *Birc) skipPrivMsg(event girc.Event) bool {
 	if event.Source.Name == b.Nick {
 		return true
 	}
+	// don't forward messages we sent via RELAYMSG
+	if relayedNick, ok := event.Tags.Get("draft/relaymsg"); ok && relayedNick == b.Nick {
+		return true
+	}
+	// This is the old name of the cap sent in spoofed messages; I've kept this in
+	// for compatibility reasons
+	if relayedNick, ok := event.Tags.Get("relaymsg"); ok && relayedNick == b.Nick {
+		return true
+	}
 	return false
 }
 

bridge/matrix/helpers.go

@@ -0,0 +1,215 @@
+package bmatrix
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"html"
+	"strings"
+	"time"
+
+	matrix "github.com/matrix-org/gomatrix"
+)
+
+func newMatrixUsername(username string) *matrixUsername {
+	mUsername := new(matrixUsername)
+
+	// check if we have a </tag>. if we have, we don't escape HTML. #696
+	if htmlTag.MatchString(username) {
+		mUsername.formatted = username
+		// remove the HTML formatting for beautiful push messages #1188
+		mUsername.plain = htmlReplacementTag.ReplaceAllString(username, "")
+	} else {
+		mUsername.formatted = html.EscapeString(username)
+		mUsername.plain = username
+	}
+
+	return mUsername
+}
+
+// getRoomID retrieves a matching room ID from the channel name.
+func (b *Bmatrix) getRoomID(channel string) string {
+	b.RLock()
+	defer b.RUnlock()
+	for ID, name := range b.RoomMap {
+		if name == channel {
+			return ID
+		}
+	}
+
+	return ""
+}
+
+// interface2Struct marshals and immediately unmarshals an interface.
+// Useful for converting map[string]interface{} to a struct.
+func interface2Struct(in interface{}, out interface{}) error {
+	jsonObj, err := json.Marshal(in)
+	if err != nil {
+		return err //nolint:wrapcheck
+	}
+
+	return json.Unmarshal(jsonObj, out)
+}
+
+// getDisplayName retrieves the displayName for mxid, querying the homserver if the mxid is not in the cache.
+func (b *Bmatrix) getDisplayName(mxid string) string {
+	if b.GetBool("UseUserName") {
+		return mxid[1:]
+	}
+
+	b.RLock()
+	if val, present := b.NicknameMap[mxid]; present {
+		b.RUnlock()
+
+		return val.displayName
+	}
+	b.RUnlock()
+
+	displayName, err := b.mc.GetDisplayName(mxid)
+	var httpError *matrix.HTTPError
+	if errors.As(err, &httpError) {
+		b.Log.Warnf("Couldn't retrieve the display name for %s", mxid)
+	}
+
+	if err != nil {
+		return b.cacheDisplayName(mxid, mxid[1:])
+	}
+
+	return b.cacheDisplayName(mxid, displayName.DisplayName)
+}
+
+// cacheDisplayName stores the mapping between a mxid and a display name, to be reused later without performing a query to the homserver.
+// Note that old entries are cleaned when this function is called.
+func (b *Bmatrix) cacheDisplayName(mxid string, displayName string) string {
+	now := time.Now()
+
+	// scan to delete old entries, to stop memory usage from becoming too high with old entries.
+	// In addition, we also detect if another user have the same username, and if so, we append their mxids to their usernames to differentiate them.
+	toDelete := []string{}
+	conflict := false
+
+	b.Lock()
+	for mxid, v := range b.NicknameMap {
+		// to prevent username reuse across matrix servers - or even on the same server, append
+		// the mxid to the username when there is a conflict
+		if v.displayName == displayName {
+			conflict = true
+			// TODO: it would be nice to be able to rename previous messages from this user.
+			// The current behavior is that only users with clashing usernames and *that have spoken since the bridge last started* will get their mxids shown, and I don't know if that's the expected behavior.
+			v.displayName = fmt.Sprintf("%s (%s)", displayName, mxid)
+			b.NicknameMap[mxid] = v
+		}
+
+		if now.Sub(v.lastUpdated) > 10*time.Minute {
+			toDelete = append(toDelete, mxid)
+		}
+	}
+
+	if conflict {
+		displayName = fmt.Sprintf("%s (%s)", displayName, mxid)
+	}
+
+	for _, v := range toDelete {
+		delete(b.NicknameMap, v)
+	}
+
+	b.NicknameMap[mxid] = NicknameCacheEntry{
+		displayName: displayName,
+		lastUpdated: now,
+	}
+	b.Unlock()
+
+	return displayName
+}
+
+// handleError converts errors into httpError.
+//nolint:exhaustivestruct
+func handleError(err error) *httpError {
+	var mErr matrix.HTTPError
+	if !errors.As(err, &mErr) {
+		return &httpError{
+			Err: "not a HTTPError",
+		}
+	}
+
+	var httpErr httpError
+
+	if err := json.Unmarshal(mErr.Contents, &httpErr); err != nil {
+		return &httpError{
+			Err: "unmarshal failed",
+		}
+	}
+
+	return &httpErr
+}
+
+func (b *Bmatrix) containsAttachment(content map[string]interface{}) bool {
+	// Skip empty messages
+	if content["msgtype"] == nil {
+		return false
+	}
+
+	// Only allow image,video or file msgtypes
+	if !(content["msgtype"].(string) == "m.image" ||
+		content["msgtype"].(string) == "m.video" ||
+		content["msgtype"].(string) == "m.file") {
+		return false
+	}
+
+	return true
+}
+
+// getAvatarURL returns the avatar URL of the specified sender.
+func (b *Bmatrix) getAvatarURL(sender string) string {
+	urlPath := b.mc.BuildURL("profile", sender, "avatar_url")
+
+	s := struct {
+		AvatarURL string `json:"avatar_url"`
+	}{}
+
+	err := b.mc.MakeRequest("GET", urlPath, nil, &s)
+	if err != nil {
+		b.Log.Errorf("getAvatarURL failed: %s", err)
+
+		return ""
+	}
+
+	url := strings.ReplaceAll(s.AvatarURL, "mxc://", b.GetString("Server")+"/_matrix/media/r0/thumbnail/")
+	if url != "" {
+		url += "?width=37&height=37&method=crop"
+	}
+
+	return url
+}
+
+// handleRatelimit handles the ratelimit errors and return if we're ratelimited and the amount of time to sleep
+func (b *Bmatrix) handleRatelimit(err error) (time.Duration, bool) {
+	httpErr := handleError(err)
+	if httpErr.Errcode != "M_LIMIT_EXCEEDED" {
+		return 0, false
+	}
+
+	b.Log.Debugf("ratelimited: %s", httpErr.Err)
+	b.Log.Infof("getting ratelimited by matrix, sleeping approx %d seconds before retrying", httpErr.RetryAfterMs/1000)
+
+	return time.Duration(httpErr.RetryAfterMs) * time.Millisecond, true
+}
+
+// retry function will check if we're ratelimited and retries again when backoff time expired
+// returns original error if not 429 ratelimit
+func (b *Bmatrix) retry(f func() error) error {
+	b.rateMutex.Lock()
+	defer b.rateMutex.Unlock()
+
+	for {
+		if err := f(); err != nil {
+			if backoff, ok := b.handleRatelimit(err); ok {
+				time.Sleep(backoff)
+			} else {
+				return err
+			}
+		} else {
+			return nil
+		}
+	}
+}

bridge/matrix/matrix.go

@@ -3,52 +3,105 @@ package bmatrix
 import (
 	"bytes"
 	"fmt"
-	"html"
 	"mime"
 	"regexp"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/42wim/matterbridge/bridge"
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/42wim/matterbridge/bridge/helper"
-	matrix "github.com/matterbridge/gomatrix"
+	matrix "github.com/matrix-org/gomatrix"
 )
 
+var (
+	htmlTag            = regexp.MustCompile("</.*?>")
+	htmlReplacementTag = regexp.MustCompile("<[^>]*>")
+)
+
+type NicknameCacheEntry struct {
+	displayName string
+	lastUpdated time.Time
+}
+
 type Bmatrix struct {
-	mc      *matrix.Client
-	UserID  string
-	RoomMap map[string]string
+	mc          *matrix.Client
+	UserID      string
+	NicknameMap map[string]NicknameCacheEntry
+	RoomMap     map[string]string
+	rateMutex   sync.RWMutex
 	sync.RWMutex
-	htmlTag *regexp.Regexp
 	*bridge.Config
 }
 
+type httpError struct {
+	Errcode      string `json:"errcode"`
+	Err          string `json:"error"`
+	RetryAfterMs int    `json:"retry_after_ms"`
+}
+
+type matrixUsername struct {
+	plain     string
+	formatted string
+}
+
+// SubTextMessage represents the new content of the message in edit messages.
+type SubTextMessage struct {
+	MsgType string `json:"msgtype"`
+	Body    string `json:"body"`
+}
+
+// MessageRelation explains how the current message relates to a previous message.
+// Notably used for message edits.
+type MessageRelation struct {
+	EventID string `json:"event_id"`
+	Type    string `json:"rel_type"`
+}
+
+type EditedMessage struct {
+	NewContent SubTextMessage  `json:"m.new_content"`
+	RelatedTo  MessageRelation `json:"m.relates_to"`
+	matrix.TextMessage
+}
+
 func New(cfg *bridge.Config) bridge.Bridger {
 	b := &Bmatrix{Config: cfg}
-	b.htmlTag = regexp.MustCompile("</.*?>")
 	b.RoomMap = make(map[string]string)
+	b.NicknameMap = make(map[string]NicknameCacheEntry)
 	return b
 }
 
 func (b *Bmatrix) Connect() error {
 	var err error
 	b.Log.Infof("Connecting %s", b.GetString("Server"))
-	b.mc, err = matrix.NewClient(b.GetString("Server"), "", "")
-	if err != nil {
-		return err
+	if b.GetString("MxID") != "" && b.GetString("Token") != "" {
+		b.mc, err = matrix.NewClient(
+			b.GetString("Server"), b.GetString("MxID"), b.GetString("Token"),
+		)
+		if err != nil {
+			return err
+		}
+		b.UserID = b.GetString("MxID")
+		b.Log.Info("Using existing Matrix credentials")
+	} else {
+		b.mc, err = matrix.NewClient(b.GetString("Server"), "", "")
+		if err != nil {
+			return err
+		}
+		resp, err := b.mc.Login(&matrix.ReqLogin{
+			Type:       "m.login.password",
+			User:       b.GetString("Login"),
+			Password:   b.GetString("Password"),
+			Identifier: matrix.NewUserIdentifier(b.GetString("Login")),
+		})
+		if err != nil {
+			return err
+		}
+		b.mc.SetCredentials(resp.UserID, resp.AccessToken)
+		b.UserID = resp.UserID
+		b.Log.Info("Connection succeeded")
 	}
-	resp, err := b.mc.Login(&matrix.ReqLogin{
-		Type:     "m.login.password",
-		User:     b.GetString("Login"),
-		Password: b.GetString("Password"),
-	})
-	if err != nil {
-		return err
-	}
-	b.mc.SetCredentials(resp.UserID, resp.AccessToken)
-	b.UserID = resp.UserID
-	b.Log.Info("Connection succeeded")
 	go b.handlematrix()
 	return nil
 }
@@ -58,14 +111,18 @@ func (b *Bmatrix) Disconnect() error {
 }
 
 func (b *Bmatrix) JoinChannel(channel config.ChannelInfo) error {
-	resp, err := b.mc.JoinRoom(channel.Name, "", nil)
-	if err != nil {
-		return err
-	}
-	b.Lock()
-	b.RoomMap[resp.RoomID] = channel.Name
-	b.Unlock()
-	return err
+	return b.retry(func() error {
+		resp, err := b.mc.JoinRoom(channel.Name, "", nil)
+		if err != nil {
+			return err
+		}
+
+		b.Lock()
+		b.RoomMap[resp.RoomID] = channel.Name
+		b.Unlock()
+
+		return nil
+	})
 }
 
 func (b *Bmatrix) Send(msg config.Message) (string, error) {
@@ -74,17 +131,30 @@ func (b *Bmatrix) Send(msg config.Message) (string, error) {
 	channel := b.getRoomID(msg.Channel)
 	b.Log.Debugf("Channel %s maps to channel id %s", msg.Channel, channel)
 
+	username := newMatrixUsername(msg.Username)
+
 	// Make a action /me of the message
 	if msg.Event == config.EventUserAction {
 		m := matrix.TextMessage{
-			MsgType: "m.emote",
-			Body:    msg.Username + msg.Text,
+			MsgType:       "m.emote",
+			Body:          username.plain + msg.Text,
+			FormattedBody: username.formatted + msg.Text,
 		}
-		resp, err := b.mc.SendMessageEvent(channel, "m.room.message", m)
-		if err != nil {
-			return "", err
-		}
-		return resp.EventID, err
+
+		msgID := ""
+
+		err := b.retry(func() error {
+			resp, err := b.mc.SendMessageEvent(channel, "m.room.message", m)
+			if err != nil {
+				return err
+			}
+
+			msgID = resp.EventID
+
+			return err
+		})
+
+		return msgID, err
 	}
 
 	// Delete message
@@ -92,17 +162,34 @@ func (b *Bmatrix) Send(msg config.Message) (string, error) {
 		if msg.ID == "" {
 			return "", nil
 		}
-		resp, err := b.mc.RedactEvent(channel, msg.ID, &matrix.ReqRedact{})
-		if err != nil {
-			return "", err
-		}
-		return resp.EventID, err
+
+		msgID := ""
+
+		err := b.retry(func() error {
+			resp, err := b.mc.RedactEvent(channel, msg.ID, &matrix.ReqRedact{})
+			if err != nil {
+				return err
+			}
+
+			msgID = resp.EventID
+
+			return err
+		})
+
+		return msgID, err
 	}
 
 	// Upload a file if it exists
 	if msg.Extra != nil {
 		for _, rmsg := range helper.HandleExtra(&msg, b.General) {
-			if _, err := b.mc.SendText(channel, rmsg.Username+rmsg.Text); err != nil {
+			rmsg := rmsg
+
+			err := b.retry(func() error {
+				_, err := b.mc.SendText(channel, rmsg.Username+rmsg.Text)
+
+				return err
+			})
+			if err != nil {
 				b.Log.Errorf("sendText failed: %s", err)
 			}
 		}
@@ -113,45 +200,105 @@ func (b *Bmatrix) Send(msg config.Message) (string, error) {
 	}
 
 	// Edit message if we have an ID
-	// matrix has no editing support
+	if msg.ID != "" {
+		rmsg := EditedMessage{TextMessage: matrix.TextMessage{
+			Body:    username.plain + msg.Text,
+			MsgType: "m.text",
+		}}
+		if b.GetBool("HTMLDisable") {
+			rmsg.TextMessage.FormattedBody = username.formatted + "* " + msg.Text
+		} else {
+			rmsg.Format = "org.matrix.custom.html"
+			rmsg.TextMessage.FormattedBody = username.formatted + "* " + helper.ParseMarkdown(msg.Text)
+		}
+		rmsg.NewContent = SubTextMessage{
+			Body:    rmsg.TextMessage.Body,
+			MsgType: "m.text",
+		}
+		rmsg.RelatedTo = MessageRelation{
+			EventID: msg.ID,
+			Type:    "m.replace",
+		}
 
-	// Use notices to send join/leave events
-	if msg.Event == config.EventJoinLeave {
-		resp, err := b.mc.SendNotice(channel, msg.Username+msg.Text)
+		err := b.retry(func() error {
+			_, err := b.mc.SendMessageEvent(channel, "m.room.message", rmsg)
+
+			return err
+		})
 		if err != nil {
 			return "", err
 		}
+
+		return msg.ID, nil
+	}
+
+	// Use notices to send join/leave events
+	if msg.Event == config.EventJoinLeave {
+		m := matrix.TextMessage{
+			MsgType:       "m.notice",
+			Body:          username.plain + msg.Text,
+			FormattedBody: username.formatted + msg.Text,
+		}
+
+		var (
+			resp *matrix.RespSendEvent
+			err  error
+		)
+
+		err = b.retry(func() error {
+			resp, err = b.mc.SendMessageEvent(channel, "m.room.message", m)
+
+			return err
+		})
+		if err != nil {
+			return "", err
+		}
+
 		return resp.EventID, err
 	}
 
-	username := html.EscapeString(msg.Username)
-	// check if we have a </tag>. if we have, we don't escape HTML. #696
-	if b.htmlTag.MatchString(msg.Username) {
-		username = msg.Username
+	if b.GetBool("HTMLDisable") {
+		var (
+			resp *matrix.RespSendEvent
+			err  error
+		)
+
+		err = b.retry(func() error {
+			resp, err = b.mc.SendText(channel, username.plain+msg.Text)
+
+			return err
+		})
+		if err != nil {
+			return "", err
+		}
+
+		return resp.EventID, err
 	}
+
 	// Post normal message with HTML support (eg riot.im)
-	resp, err := b.mc.SendHTML(channel, msg.Username+msg.Text, username+helper.ParseMarkdown(msg.Text))
+	var (
+		resp *matrix.RespSendEvent
+		err  error
+	)
+
+	err = b.retry(func() error {
+		resp, err = b.mc.SendFormattedText(channel, username.plain+msg.Text,
+			username.formatted+helper.ParseMarkdown(msg.Text))
+
+		return err
+	})
 	if err != nil {
 		return "", err
 	}
-	return resp.EventID, err
-}
 
-func (b *Bmatrix) getRoomID(channel string) string {
-	b.RLock()
-	defer b.RUnlock()
-	for ID, name := range b.RoomMap {
-		if name == channel {
-			return ID
-		}
-	}
-	return ""
+	return resp.EventID, err
 }
 
 func (b *Bmatrix) handlematrix() {
 	syncer := b.mc.Syncer.(*matrix.DefaultSyncer)
 	syncer.OnEventType("m.room.redaction", b.handleEvent)
 	syncer.OnEventType("m.room.message", b.handleEvent)
+	syncer.OnEventType("m.room.member", b.handleMemberChange)
 	go func() {
 		for {
 			if err := b.mc.Sync(); err != nil {
@@ -161,6 +308,45 @@ func (b *Bmatrix) handlematrix() {
 	}()
 }
 
+func (b *Bmatrix) handleEdit(ev *matrix.Event, rmsg config.Message) bool {
+	relationInterface, present := ev.Content["m.relates_to"]
+	newContentInterface, present2 := ev.Content["m.new_content"]
+	if !(present && present2) {
+		return false
+	}
+
+	var relation MessageRelation
+	if err := interface2Struct(relationInterface, &relation); err != nil {
+		b.Log.Warnf("Couldn't parse 'm.relates_to' object with value %#v", relationInterface)
+		return false
+	}
+
+	var newContent SubTextMessage
+	if err := interface2Struct(newContentInterface, &newContent); err != nil {
+		b.Log.Warnf("Couldn't parse 'm.new_content' object with value %#v", newContentInterface)
+		return false
+	}
+
+	if relation.Type != "m.replace" {
+		return false
+	}
+
+	rmsg.ID = relation.EventID
+	rmsg.Text = newContent.Body
+	b.Remote <- rmsg
+
+	return true
+}
+
+func (b *Bmatrix) handleMemberChange(ev *matrix.Event) {
+	// Update the displayname on join messages, according to https://matrix.org/docs/spec/client_server/r0.6.1#events-on-change-of-profile-information
+	if ev.Content["membership"] == "join" {
+		if dn, ok := ev.Content["displayname"].(string); ok {
+			b.cacheDisplayName(ev.Sender, dn)
+		}
+	}
+}
+
 func (b *Bmatrix) handleEvent(ev *matrix.Event) {
 	b.Log.Debugf("== Receiving event: %#v", ev)
 	if ev.Sender != b.UserID {
@@ -174,7 +360,7 @@ func (b *Bmatrix) handleEvent(ev *matrix.Event) {
 
 		// Create our message
 		rmsg := config.Message{
-			Username: ev.Sender[1:],
+			Username: b.getDisplayName(ev.Sender),
 			Channel:  channel,
 			Account:  b.Account,
 			UserID:   ev.Sender,
@@ -182,13 +368,6 @@ func (b *Bmatrix) handleEvent(ev *matrix.Event) {
 			Avatar:   b.getAvatarURL(ev.Sender),
 		}
 
-		// Text must be a string
-		if rmsg.Text, ok = ev.Content["body"].(string); !ok {
-			b.Log.Errorf("Content[body] is not a string: %T\n%#v",
-				ev.Content["body"], ev.Content)
-			return
-		}
-
 		// Remove homeserver suffix if configured
 		if b.GetBool("NoHomeServerSuffix") {
 			re := regexp.MustCompile("(.*?):.*")
@@ -204,11 +383,23 @@ func (b *Bmatrix) handleEvent(ev *matrix.Event) {
 			return
 		}
 
+		// Text must be a string
+		if rmsg.Text, ok = ev.Content["body"].(string); !ok {
+			b.Log.Errorf("Content[body] is not a string: %T\n%#v",
+				ev.Content["body"], ev.Content)
+			return
+		}
+
 		// Do we have a /me action
 		if ev.Content["msgtype"].(string) == "m.emote" {
 			rmsg.Event = config.EventUserAction
 		}
 
+		// Is it an edit?
+		if b.handleEdit(ev, rmsg) {
+			return
+		}
+
 		// Do we have attachments
 		if b.containsAttachment(ev.Content) {
 			err := b.handleDownloadFile(&rmsg, ev.Content)
@@ -219,6 +410,11 @@ func (b *Bmatrix) handleEvent(ev *matrix.Event) {
 
 		b.Log.Debugf("<= Sending message from %s on %s to gateway", ev.Sender, b.Account)
 		b.Remote <- rmsg
+
+		// not crucial, so no ratelimit check here
+		if err := b.mc.MarkRead(ev.RoomID, ev.ID); err != nil {
+			b.Log.Errorf("couldn't mark message as read %s", err.Error())
+		}
 	}
 }
 
@@ -293,27 +489,30 @@ func (b *Bmatrix) handleUploadFiles(msg *config.Message, channel string) (string
 
 // handleUploadFile handles native upload of a file.
 func (b *Bmatrix) handleUploadFile(msg *config.Message, channel string, fi *config.FileInfo) {
+	username := newMatrixUsername(msg.Username)
 	content := bytes.NewReader(*fi.Data)
 	sp := strings.Split(fi.Name, ".")
 	mtype := mime.TypeByExtension("." + sp[len(sp)-1])
-	if !(strings.Contains(mtype, "image") || strings.Contains(mtype, "video") ||
-		strings.Contains(mtype, "application") || strings.Contains(mtype, "audio")) {
-		return
-	}
-	if fi.Comment != "" {
-		_, err := b.mc.SendText(channel, msg.Username+fi.Comment)
-		if err != nil {
-			b.Log.Errorf("file comment failed: %#v", err)
-		}
-	} else {
-		// image and video uploads send no username, we have to do this ourself here #715
-		_, err := b.mc.SendText(channel, msg.Username)
-		if err != nil {
-			b.Log.Errorf("file comment failed: %#v", err)
-		}
+	// image and video uploads send no username, we have to do this ourself here #715
+	err := b.retry(func() error {
+		_, err := b.mc.SendFormattedText(channel, username.plain+fi.Comment, username.formatted+fi.Comment)
+
+		return err
+	})
+	if err != nil {
+		b.Log.Errorf("file comment failed: %#v", err)
 	}
+
 	b.Log.Debugf("uploading file: %s %s", fi.Name, mtype)
-	res, err := b.mc.UploadToContentRepo(content, mtype, int64(len(*fi.Data)))
+
+	var res *matrix.RespMediaUpload
+
+	err = b.retry(func() error {
+		res, err = b.mc.UploadToContentRepo(content, mtype, int64(len(*fi.Data)))
+
+		return err
+	})
+
 	if err != nil {
 		b.Log.Errorf("file upload failed: %#v", err)
 		return
@@ -322,56 +521,60 @@ func (b *Bmatrix) handleUploadFile(msg *config.Message, channel string, fi *conf
 	switch {
 	case strings.Contains(mtype, "video"):
 		b.Log.Debugf("sendVideo %s", res.ContentURI)
-		_, err = b.mc.SendVideo(channel, fi.Name, res.ContentURI)
+		err = b.retry(func() error {
+			_, err = b.mc.SendVideo(channel, fi.Name, res.ContentURI)
+
+			return err
+		})
 		if err != nil {
 			b.Log.Errorf("sendVideo failed: %#v", err)
 		}
 	case strings.Contains(mtype, "image"):
 		b.Log.Debugf("sendImage %s", res.ContentURI)
-		_, err = b.mc.SendImage(channel, fi.Name, res.ContentURI)
+		err = b.retry(func() error {
+			_, err = b.mc.SendImage(channel, fi.Name, res.ContentURI)
+
+			return err
+		})
 		if err != nil {
 			b.Log.Errorf("sendImage failed: %#v", err)
 		}
-	case strings.Contains(mtype, "application"):
-		b.Log.Debugf("sendFile %s", res.ContentURI)
-		_, err = b.mc.SendFile(channel, fi.Name, res.ContentURI, mtype, uint(len(*fi.Data)))
-		if err != nil {
-			b.Log.Errorf("sendFile failed: %#v", err)
-		}
 	case strings.Contains(mtype, "audio"):
 		b.Log.Debugf("sendAudio %s", res.ContentURI)
-		_, err = b.mc.SendAudio(channel, fi.Name, res.ContentURI, mtype, uint(len(*fi.Data)))
+		err = b.retry(func() error {
+			_, err = b.mc.SendMessageEvent(channel, "m.room.message", matrix.AudioMessage{
+				MsgType: "m.audio",
+				Body:    fi.Name,
+				URL:     res.ContentURI,
+				Info: matrix.AudioInfo{
+					Mimetype: mtype,
+					Size:     uint(len(*fi.Data)),
+				},
+			})
+
+			return err
+		})
 		if err != nil {
 			b.Log.Errorf("sendAudio failed: %#v", err)
 		}
+	default:
+		b.Log.Debugf("sendFile %s", res.ContentURI)
+		err = b.retry(func() error {
+			_, err = b.mc.SendMessageEvent(channel, "m.room.message", matrix.FileMessage{
+				MsgType: "m.file",
+				Body:    fi.Name,
+				URL:     res.ContentURI,
+				Info: matrix.FileInfo{
+					Mimetype: mtype,
+					Size:     uint(len(*fi.Data)),
+				},
+			})
+
+			return err
+		})
+		if err != nil {
+			b.Log.Errorf("sendFile failed: %#v", err)
+		}
 	}
 	b.Log.Debugf("result: %#v", res)
 }
-
-// skipMessages returns true if this message should not be handled
-func (b *Bmatrix) containsAttachment(content map[string]interface{}) bool {
-	// Skip empty messages
-	if content["msgtype"] == nil {
-		return false
-	}
-
-	// Only allow image,video or file msgtypes
-	if !(content["msgtype"].(string) == "m.image" ||
-		content["msgtype"].(string) == "m.video" ||
-		content["msgtype"].(string) == "m.file") {
-		return false
-	}
-	return true
-}
-
-// getAvatarURL returns the avatar URL of the specified sender
-func (b *Bmatrix) getAvatarURL(sender string) string {
-	mxcURL, err := b.mc.GetSenderAvatarURL(sender)
-	if err != nil {
-		b.Log.Errorf("getAvatarURL failed: %s", err)
-		return ""
-	}
-	url := strings.ReplaceAll(mxcURL, "mxc://", b.GetString("Server")+"/_matrix/media/r0/thumbnail/")
-	url += "?width=37&height=37&method=crop"
-	return url
-}

bridge/matrix/matrix_test.go

@@ -0,0 +1,28 @@
+package bmatrix
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestPlainUsername(t *testing.T) {
+	uut := newMatrixUsername("MyUser")
+
+	assert.Equal(t, "MyUser", uut.formatted)
+	assert.Equal(t, "MyUser", uut.plain)
+}
+
+func TestHTMLUsername(t *testing.T) {
+	uut := newMatrixUsername("<b>MyUser</b>")
+
+	assert.Equal(t, "<b>MyUser</b>", uut.formatted)
+	assert.Equal(t, "MyUser", uut.plain)
+}
+
+func TestFancyUsername(t *testing.T) {
+	uut := newMatrixUsername("<MyUser>")
+
+	assert.Equal(t, "&lt;MyUser&gt;", uut.formatted)
+	assert.Equal(t, "<MyUser>", uut.plain)
+}

bridge/mattermost/handlers.go

@@ -4,7 +4,7 @@ import (
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/42wim/matterbridge/bridge/helper"
 	"github.com/42wim/matterbridge/matterclient"
-	"github.com/mattermost/mattermost-server/model"
+	"github.com/mattermost/mattermost-server/v5/model"
 )
 
 // handleDownloadAvatar downloads the avatar of userid from channel
@@ -108,7 +108,7 @@ func (b *Bmattermost) handleMatterClient(messages chan *config.Message) {
 			Channel:  message.Channel,
 			Text:     message.Text,
 			ID:       message.Post.Id,
-			ParentID: message.Post.ParentId,
+			ParentID: message.Post.RootId, // ParentID is obsolete with mattermost
 			Extra:    make(map[string][]interface{}),
 		}
 

bridge/mattermost/helpers.go

@@ -7,7 +7,7 @@ import (
 	"github.com/42wim/matterbridge/bridge/helper"
 	"github.com/42wim/matterbridge/matterclient"
 	"github.com/42wim/matterbridge/matterhook"
-	"github.com/mattermost/mattermost-server/model"
+	"github.com/mattermost/mattermost-server/v5/model"
 )
 
 func (b *Bmattermost) doConnectWebhookBind() error {

bridge/mattermost/mattermost.go

@@ -122,11 +122,20 @@ func (b *Bmattermost) Send(msg config.Message) (string, error) {
 	}
 
 	// Handle prefix hint for unthreaded messages.
-	if msg.ParentID == "msg-parent-not-found" {
+	if msg.ParentNotFound() {
 		msg.ParentID = ""
 		msg.Text = fmt.Sprintf("[thread]: %s", msg.Text)
 	}
 
+	// we only can reply to the root of the thread, not to a specific ID (like discord for example does)
+	if msg.ParentID != "" {
+		post, res := b.mc.Client.GetPost(msg.ParentID, "")
+		if res.Error != nil {
+			b.Log.Errorf("getting post %s failed: %s", msg.ParentID, res.Error.DetailedError)
+		}
+		msg.ParentID = post.RootId
+	}
+
 	// Upload a file if it exists
 	if msg.Extra != nil {
 		for _, rmsg := range helper.HandleExtra(&msg, b.General) {

bridge/msteams/handler.go

@@ -9,7 +9,7 @@ import (
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/42wim/matterbridge/bridge/helper"
 
-	msgraph "github.com/matterbridge/msgraph.go/beta"
+	msgraph "github.com/yaegashi/msgraph.go/beta"
 )
 
 func (b *Bmsteams) findFile(weburl string) (string, error) {

bridge/msteams/msteams.go

@@ -10,11 +10,11 @@ import (
 
 	"github.com/42wim/matterbridge/bridge"
 	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/davecgh/go-spew/spew"
 
-	//	"github.com/davecgh/go-spew/spew"
-	msgraph "github.com/matterbridge/msgraph.go/beta"
-	"github.com/matterbridge/msgraph.go/msauth"
 	"github.com/mattn/godown"
+	msgraph "github.com/yaegashi/msgraph.go/beta"
+	"github.com/yaegashi/msgraph.go/msauth"
 
 	"golang.org/x/oauth2"
 )
@@ -72,19 +72,30 @@ func (b *Bmsteams) Disconnect() error {
 }
 
 func (b *Bmsteams) JoinChannel(channel config.ChannelInfo) error {
-	go b.poll(channel.Name)
+	go func(name string) {
+		for {
+			err := b.poll(name)
+			if err != nil {
+				b.Log.Errorf("polling failed for %s: %s. retrying in 5 seconds", name, err)
+			}
+			time.Sleep(time.Second * 5)
+		}
+	}(channel.Name)
 	return nil
 }
 
 func (b *Bmsteams) Send(msg config.Message) (string, error) {
 	b.Log.Debugf("=> Receiving %#v", msg)
-	if msg.ParentID != "" && msg.ParentID != "msg-parent-not-found" {
+	if msg.ParentValid() {
 		return b.sendReply(msg)
 	}
-	if msg.ParentID == "msg-parent-not-found" {
+
+	// Handle prefix hint for unthreaded messages.
+	if msg.ParentNotFound() {
 		msg.ParentID = ""
 		msg.Text = fmt.Sprintf("[thread]: %s", msg.Text)
 	}
+
 	ct := b.gc.Teams().ID(b.GetString("TeamID")).Channels().ID(msg.Channel).Messages().Request()
 	text := msg.Username + msg.Text
 	content := &msgraph.ItemBody{Content: &text}
@@ -121,12 +132,12 @@ func (b *Bmsteams) getMessages(channel string) ([]msgraph.ChatMessage, error) {
 }
 
 //nolint:gocognit
-func (b *Bmsteams) poll(channelName string) {
+func (b *Bmsteams) poll(channelName string) error {
 	msgmap := make(map[string]time.Time)
 	b.Log.Debug("getting initial messages")
 	res, err := b.getMessages(channelName)
 	if err != nil {
-		panic(err)
+		return err
 	}
 	for _, msg := range res {
 		msgmap[*msg.ID] = *msg.CreatedDateTime
@@ -139,7 +150,7 @@ func (b *Bmsteams) poll(channelName string) {
 	for {
 		res, err := b.getMessages(channelName)
 		if err != nil {
-			panic(err)
+			return err
 		}
 		for i := len(res) - 1; i >= 0; i-- {
 			msg := res[i]
@@ -151,11 +162,22 @@ func (b *Bmsteams) poll(channelName string) {
 					continue
 				}
 			}
+
+			if b.GetBool("debug") {
+				b.Log.Debug("Msg dump: ", spew.Sdump(msg))
+			}
+
+			// skip non-user message for now.
+			if msg.From.User == nil {
+				continue
+			}
+
 			if *msg.From.User.ID == b.botID {
 				b.Log.Debug("skipping own message")
 				msgmap[*msg.ID] = *msg.CreatedDateTime
 				continue
 			}
+
 			msgmap[*msg.ID] = *msg.CreatedDateTime
 			if msg.LastModifiedDateTime != nil {
 				msgmap[*msg.ID] = *msg.LastModifiedDateTime

bridge/mumble/handlers.go

@@ -0,0 +1,96 @@
+package bmumble
+
+import (
+	"strconv"
+	"time"
+
+	"layeh.com/gumble/gumble"
+
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/42wim/matterbridge/bridge/helper"
+)
+
+func (b *Bmumble) handleServerConfig(event *gumble.ServerConfigEvent) {
+	b.serverConfigUpdate <- *event
+}
+
+func (b *Bmumble) handleTextMessage(event *gumble.TextMessageEvent) {
+	sender := "unknown"
+	if event.TextMessage.Sender != nil {
+		sender = event.TextMessage.Sender.Name
+	}
+	// If the text message is received before receiving a ServerSync
+	// and UserState, Client.Self or Self.Channel are nil
+	if event.Client.Self == nil || event.Client.Self.Channel == nil {
+		b.Log.Warn("Connection bootstrap not finished, discarding text message")
+		return
+	}
+	// Convert Mumble HTML messages to markdown
+	parts, err := b.convertHTMLtoMarkdown(event.TextMessage.Message)
+	if err != nil {
+		b.Log.Error(err)
+	}
+	now := time.Now().UTC()
+	for i, part := range parts {
+		// Construct matterbridge message and pass on to the gateway
+		rmsg := config.Message{
+			Channel:  strconv.FormatUint(uint64(event.Client.Self.Channel.ID), 10),
+			Username: sender,
+			UserID:   sender + "@" + b.Host,
+			Account:  b.Account,
+		}
+		if part.Image == nil {
+			rmsg.Text = part.Text
+		} else {
+			fname := b.Account + "_" + strconv.FormatInt(now.UnixNano(), 10) + "_" + strconv.Itoa(i) + part.FileExtension
+			rmsg.Extra = make(map[string][]interface{})
+			if err = helper.HandleDownloadSize(b.Log, &rmsg, fname, int64(len(part.Image)), b.General); err != nil {
+				b.Log.WithError(err).Warn("not including image in message")
+				continue
+			}
+			helper.HandleDownloadData(b.Log, &rmsg, fname, "", "", &part.Image, b.General)
+		}
+		b.Log.Debugf("Sending message to gateway: %+v", rmsg)
+		b.Remote <- rmsg
+	}
+}
+
+func (b *Bmumble) handleConnect(event *gumble.ConnectEvent) {
+	// Set the user's "bio"/comment
+	if comment := b.GetString("UserComment"); comment != "" && event.Client.Self != nil {
+		event.Client.Self.SetComment(comment)
+	}
+	// No need to talk or listen
+	event.Client.Self.SetSelfDeafened(true)
+	event.Client.Self.SetSelfMuted(true)
+	// if the Channel variable is set, this is a reconnect -> rejoin channel
+	if b.Channel != nil {
+		if err := b.doJoin(event.Client, *b.Channel); err != nil {
+			b.Log.Error(err)
+		}
+		b.Remote <- config.Message{
+			Username: "system",
+			Text:     "rejoin",
+			Channel:  "",
+			Account:  b.Account,
+			Event:    config.EventRejoinChannels,
+		}
+	}
+}
+
+func (b *Bmumble) handleUserChange(event *gumble.UserChangeEvent) {
+	// Only care about changes to self
+	if event.User != event.Client.Self {
+		return
+	}
+	// Someone attempted to move the user out of the configured channel; attempt to join back
+	if b.Channel != nil {
+		if err := b.doJoin(event.Client, *b.Channel); err != nil {
+			b.Log.Error(err)
+		}
+	}
+}
+
+func (b *Bmumble) handleDisconnect(event *gumble.DisconnectEvent) {
+	b.connected <- *event
+}

bridge/mumble/helpers.go

@@ -0,0 +1,143 @@
+package bmumble
+
+import (
+	"fmt"
+	"mime"
+	"net/http"
+	"regexp"
+	"strings"
+
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/mattn/godown"
+	"github.com/vincent-petithory/dataurl"
+)
+
+type MessagePart struct {
+	Text          string
+	FileExtension string
+	Image         []byte
+}
+
+func (b *Bmumble) decodeImage(uri string, parts *[]MessagePart) error {
+	// Decode the data:image/... URI
+	image, err := dataurl.DecodeString(uri)
+	if err != nil {
+		b.Log.WithError(err).Info("No image extracted")
+		return err
+	}
+	// Determine the file extensions for that image
+	ext, err := mime.ExtensionsByType(image.MediaType.ContentType())
+	if err != nil || len(ext) == 0 {
+		b.Log.WithError(err).Infof("No file extension registered for MIME type '%s'", image.MediaType.ContentType())
+		return err
+	}
+	// Add the image to the MessagePart slice
+	*parts = append(*parts, MessagePart{"", ext[0], image.Data})
+	return nil
+}
+
+func (b *Bmumble) tokenize(t *string) ([]MessagePart, error) {
+	// `^(.*?)` matches everything before the image
+	// `!\[[^\]]*\]\(` matches the `![alt](` part of markdown images
+	// `(data:image\/[^)]+)` matches the data: URI used by Mumble
+	// `\)` matches the closing parenthesis after the URI
+	// `(.*)$` matches the remaining text to be examined in the next iteration
+	p := regexp.MustCompile(`^(?ms)(.*?)!\[[^\]]*\]\((data:image\/[^)]+)\)(.*)$`)
+	remaining := *t
+	var parts []MessagePart
+	for {
+		tokens := p.FindStringSubmatch(remaining)
+		if tokens == nil {
+			// no match -> remaining string is non-image text
+			pre := strings.TrimSpace(remaining)
+			if len(pre) > 0 {
+				parts = append(parts, MessagePart{pre, "", nil})
+			}
+			return parts, nil
+		}
+
+		// tokens[1] is the text before the image
+		if len(tokens[1]) > 0 {
+			pre := strings.TrimSpace(tokens[1])
+			parts = append(parts, MessagePart{pre, "", nil})
+		}
+		// tokens[2] is the image URL
+		uri, err := dataurl.UnescapeToString(strings.TrimSpace(strings.ReplaceAll(tokens[2], " ", "")))
+		if err != nil {
+			b.Log.WithError(err).Info("URL unescaping failed")
+			remaining = strings.TrimSpace(tokens[3])
+			continue
+		}
+		err = b.decodeImage(uri, &parts)
+		if err != nil {
+			b.Log.WithError(err).Info("Decoding the image failed")
+		}
+		// tokens[3] is the text after the image, processed in the next iteration
+		remaining = strings.TrimSpace(tokens[3])
+	}
+}
+
+func (b *Bmumble) convertHTMLtoMarkdown(html string) ([]MessagePart, error) {
+	var sb strings.Builder
+	err := godown.Convert(&sb, strings.NewReader(html), nil)
+	if err != nil {
+		return nil, err
+	}
+	markdown := sb.String()
+	b.Log.Debugf("### to markdown: %s", markdown)
+	return b.tokenize(&markdown)
+}
+
+func (b *Bmumble) extractFiles(msg *config.Message) []config.Message {
+	var messages []config.Message
+	if msg.Extra == nil || len(msg.Extra["file"]) == 0 {
+		return messages
+	}
+	// Create a separate message for each file
+	for _, f := range msg.Extra["file"] {
+		fi := f.(config.FileInfo)
+		imsg := config.Message{
+			Channel:   msg.Channel,
+			Username:  msg.Username,
+			UserID:    msg.UserID,
+			Account:   msg.Account,
+			Protocol:  msg.Protocol,
+			Timestamp: msg.Timestamp,
+			Event:     "mumble_image",
+		}
+		// If no data is present for the file, send a link instead
+		if fi.Data == nil || len(*fi.Data) == 0 {
+			if len(fi.URL) > 0 {
+				imsg.Text = fmt.Sprintf(`<a href="%s">%s</a>`, fi.URL, fi.URL)
+				messages = append(messages, imsg)
+			} else {
+				b.Log.Infof("Not forwarding file without local data")
+			}
+			continue
+		}
+		mimeType := http.DetectContentType(*fi.Data)
+		// Mumble only supports images natively, send a link instead
+		if !strings.HasPrefix(mimeType, "image/") {
+			if len(fi.URL) > 0 {
+				imsg.Text = fmt.Sprintf(`<a href="%s">%s</a>`, fi.URL, fi.URL)
+				messages = append(messages, imsg)
+			} else {
+				b.Log.Infof("Not forwarding file of type %s", mimeType)
+			}
+			continue
+		}
+		mimeType = strings.TrimSpace(strings.Split(mimeType, ";")[0])
+		// Build data:image/...;base64,... style image URL and embed image directly into the message
+		du := dataurl.New(*fi.Data, mimeType)
+		dataURL, err := du.MarshalText()
+		if err != nil {
+			b.Log.WithError(err).Infof("Image Serialization into data URL failed (type: %s, length: %d)", mimeType, len(*fi.Data))
+			continue
+		}
+		imsg.Text = fmt.Sprintf(`<img src="%s"/>`, dataURL)
+		messages = append(messages, imsg)
+	}
+	// Remove files from original message
+	msg.Extra["file"] = nil
+	return messages
+}

bridge/mumble/mumble.go

@@ -0,0 +1,259 @@
+package bmumble
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"strconv"
+	"time"
+
+	"layeh.com/gumble/gumble"
+	"layeh.com/gumble/gumbleutil"
+
+	"github.com/42wim/matterbridge/bridge"
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/42wim/matterbridge/bridge/helper"
+	stripmd "github.com/writeas/go-strip-markdown"
+
+	// We need to import the 'data' package as an implicit dependency.
+	// See: https://godoc.org/github.com/paulrosania/go-charset/charset
+	_ "github.com/paulrosania/go-charset/data"
+)
+
+type Bmumble struct {
+	client             *gumble.Client
+	Nick               string
+	Host               string
+	Channel            *uint32
+	local              chan config.Message
+	running            chan error
+	connected          chan gumble.DisconnectEvent
+	serverConfigUpdate chan gumble.ServerConfigEvent
+	serverConfig       gumble.ServerConfigEvent
+	tlsConfig          tls.Config
+
+	*bridge.Config
+}
+
+func New(cfg *bridge.Config) bridge.Bridger {
+	b := &Bmumble{}
+	b.Config = cfg
+	b.Nick = b.GetString("Nick")
+	b.local = make(chan config.Message)
+	b.running = make(chan error)
+	b.connected = make(chan gumble.DisconnectEvent)
+	b.serverConfigUpdate = make(chan gumble.ServerConfigEvent)
+	return b
+}
+
+func (b *Bmumble) Connect() error {
+	b.Log.Infof("Connecting %s", b.GetString("Server"))
+	host, portstr, err := net.SplitHostPort(b.GetString("Server"))
+	if err != nil {
+		return err
+	}
+	b.Host = host
+	_, err = strconv.Atoi(portstr)
+	if err != nil {
+		return err
+	}
+
+	if err = b.buildTLSConfig(); err != nil {
+		return err
+	}
+
+	go b.doSend()
+	go b.connectLoop()
+	err = <-b.running
+	return err
+}
+
+func (b *Bmumble) Disconnect() error {
+	return b.client.Disconnect()
+}
+
+func (b *Bmumble) JoinChannel(channel config.ChannelInfo) error {
+	cid, err := strconv.ParseUint(channel.Name, 10, 32)
+	if err != nil {
+		return err
+	}
+	channelID := uint32(cid)
+	if b.Channel != nil && *b.Channel != channelID {
+		b.Log.Fatalf("Cannot join channel ID '%d', already joined to channel ID %d", channelID, *b.Channel)
+		return errors.New("the Mumble bridge can only join a single channel")
+	}
+	b.Channel = &channelID
+	return b.doJoin(b.client, channelID)
+}
+
+func (b *Bmumble) Send(msg config.Message) (string, error) {
+	// Only process text messages
+	b.Log.Debugf("=> Received local message %#v", msg)
+	if msg.Event != "" && msg.Event != config.EventUserAction {
+		return "", nil
+	}
+
+	attachments := b.extractFiles(&msg)
+	b.local <- msg
+	for _, a := range attachments {
+		b.local <- a
+	}
+	return "", nil
+}
+
+func (b *Bmumble) buildTLSConfig() error {
+	b.tlsConfig = tls.Config{}
+	// Load TLS client certificate keypair required for registered user authentication
+	if cpath := b.GetString("TLSClientCertificate"); cpath != "" {
+		if ckey := b.GetString("TLSClientKey"); ckey != "" {
+			cert, err := tls.LoadX509KeyPair(cpath, ckey)
+			if err != nil {
+				return err
+			}
+			b.tlsConfig.Certificates = []tls.Certificate{cert}
+		}
+	}
+	// Load TLS CA used for server verification.  If not provided, the Go system trust anchor is used
+	if capath := b.GetString("TLSCACertificate"); capath != "" {
+		ca, err := ioutil.ReadFile(capath)
+		if err != nil {
+			return err
+		}
+		b.tlsConfig.RootCAs = x509.NewCertPool()
+		b.tlsConfig.RootCAs.AppendCertsFromPEM(ca)
+	}
+	b.tlsConfig.InsecureSkipVerify = b.GetBool("SkipTLSVerify")
+	return nil
+}
+
+func (b *Bmumble) connectLoop() {
+	firstConnect := true
+	for {
+		err := b.doConnect()
+		if firstConnect {
+			b.running <- err
+		}
+		if err != nil {
+			b.Log.Errorf("Connection to server failed: %#v", err)
+			if firstConnect {
+				break
+			} else {
+				b.Log.Info("Retrying in 10s")
+				time.Sleep(10 * time.Second)
+				continue
+			}
+		}
+		firstConnect = false
+		d := <-b.connected
+		switch d.Type {
+		case gumble.DisconnectError:
+			b.Log.Errorf("Lost connection to the server (%s), attempting reconnect", d.String)
+			continue
+		case gumble.DisconnectKicked:
+			b.Log.Errorf("Kicked from the server (%s), attempting reconnect", d.String)
+			continue
+		case gumble.DisconnectBanned:
+			b.Log.Errorf("Banned from the server (%s), not attempting reconnect", d.String)
+			close(b.connected)
+			close(b.running)
+			return
+		case gumble.DisconnectUser:
+			b.Log.Infof("Disconnect successful")
+			close(b.connected)
+			close(b.running)
+			return
+		}
+	}
+}
+
+func (b *Bmumble) doConnect() error {
+	// Create new gumble config and attach event handlers
+	gumbleConfig := gumble.NewConfig()
+	gumbleConfig.Attach(gumbleutil.Listener{
+		ServerConfig: b.handleServerConfig,
+		TextMessage:  b.handleTextMessage,
+		Connect:      b.handleConnect,
+		Disconnect:   b.handleDisconnect,
+		UserChange:   b.handleUserChange,
+	})
+	gumbleConfig.Username = b.GetString("Nick")
+	if password := b.GetString("Password"); password != "" {
+		gumbleConfig.Password = password
+	}
+
+	client, err := gumble.DialWithDialer(new(net.Dialer), b.GetString("Server"), gumbleConfig, &b.tlsConfig)
+	if err != nil {
+		return err
+	}
+	b.client = client
+	return nil
+}
+
+func (b *Bmumble) doJoin(client *gumble.Client, channelID uint32) error {
+	channel, ok := client.Channels[channelID]
+	if !ok {
+		return fmt.Errorf("no channel with ID %d", channelID)
+	}
+	client.Self.Move(channel)
+	return nil
+}
+
+func (b *Bmumble) doSend() {
+	// Message sending loop that makes sure server-side
+	// restrictions and client-side message traits don't conflict
+	// with each other.
+	for {
+		select {
+		case serverConfig := <-b.serverConfigUpdate:
+			b.Log.Debugf("Received server config update: AllowHTML=%#v, MaximumMessageLength=%#v", serverConfig.AllowHTML, serverConfig.MaximumMessageLength)
+			b.serverConfig = serverConfig
+		case msg := <-b.local:
+			b.processMessage(&msg)
+		}
+	}
+}
+
+func (b *Bmumble) processMessage(msg *config.Message) {
+	b.Log.Debugf("Processing message %s", msg.Text)
+
+	allowHTML := true
+	if b.serverConfig.AllowHTML != nil {
+		allowHTML = *b.serverConfig.AllowHTML
+	}
+
+	// If this is a specially generated image message, send it unmodified
+	if msg.Event == "mumble_image" {
+		if allowHTML {
+			b.client.Self.Channel.Send(msg.Username+msg.Text, false)
+		} else {
+			b.Log.Info("Can't send image, server does not allow HTML messages")
+		}
+		return
+	}
+
+	// Don't process empty messages
+	if len(msg.Text) == 0 {
+		return
+	}
+	// If HTML is allowed, convert markdown into HTML, otherwise strip markdown
+	if allowHTML {
+		msg.Text = helper.ParseMarkdown(msg.Text)
+	} else {
+		msg.Text = stripmd.Strip(msg.Text)
+	}
+
+	// If there is a maximum message length, split and truncate the lines
+	var msgLines []string
+	if maxLength := b.serverConfig.MaximumMessageLength; maxLength != nil {
+		msgLines = helper.GetSubLines(msg.Text, *maxLength-len(msg.Username), b.GetString("MessageClipped"))
+	} else {
+		msgLines = helper.GetSubLines(msg.Text, 0, b.GetString("MessageClipped"))
+	}
+	// Send the individual lindes
+	for i := range msgLines {
+		b.client.Self.Channel.Send(msg.Username+msgLines[i], false)
+	}
+}

bridge/nctalk/nctalk.go

@@ -0,0 +1,284 @@
+package nctalk
+
+import (
+	"context"
+	"crypto/tls"
+	"strconv"
+	"strings"
+
+	"github.com/42wim/matterbridge/bridge"
+	"github.com/42wim/matterbridge/bridge/config"
+
+	"gomod.garykim.dev/nc-talk/ocs"
+	"gomod.garykim.dev/nc-talk/room"
+	"gomod.garykim.dev/nc-talk/user"
+)
+
+type Btalk struct {
+	user  *user.TalkUser
+	rooms []Broom
+	*bridge.Config
+}
+
+func New(cfg *bridge.Config) bridge.Bridger {
+	return &Btalk{Config: cfg}
+}
+
+type Broom struct {
+	room      *room.TalkRoom
+	ctx       context.Context
+	ctxCancel context.CancelFunc
+}
+
+func (b *Btalk) Connect() error {
+	b.Log.Info("Connecting")
+	tconfig := &user.TalkUserConfig{
+		TLSConfig: &tls.Config{
+			InsecureSkipVerify: b.GetBool("SkipTLSVerify"), //nolint:gosec
+		},
+	}
+	var err error
+	b.user, err = user.NewUser(b.GetString("Server"), b.GetString("Login"), b.GetString("Password"), tconfig)
+	if err != nil {
+		b.Log.Error("Config could not be used")
+		return err
+	}
+	_, err = b.user.Capabilities()
+	if err != nil {
+		b.Log.Error("Cannot Connect")
+		return err
+	}
+	b.Log.Info("Connected")
+	return nil
+}
+
+func (b *Btalk) Disconnect() error {
+	for _, r := range b.rooms {
+		r.ctxCancel()
+	}
+	return nil
+}
+
+func (b *Btalk) JoinChannel(channel config.ChannelInfo) error {
+	tr, err := room.NewTalkRoom(b.user, channel.Name)
+	if err != nil {
+		return err
+	}
+	newRoom := Broom{
+		room: tr,
+	}
+	newRoom.ctx, newRoom.ctxCancel = context.WithCancel(context.Background())
+	c, err := newRoom.room.ReceiveMessages(newRoom.ctx)
+	if err != nil {
+		return err
+	}
+	b.rooms = append(b.rooms, newRoom)
+
+	go func() {
+		for msg := range c {
+			msg := msg
+
+			if msg.Error != nil {
+				b.Log.Errorf("Fatal message poll error: %s\n", msg.Error)
+
+				return
+			}
+
+			// Ignore messages that are from the bot user
+			if msg.ActorID == b.user.User {
+				continue
+			}
+
+			// Handle deleting messages
+			if msg.MessageType == ocs.MessageSystem && msg.Parent != nil && msg.Parent.MessageType == ocs.MessageDelete {
+				b.handleDeletingMessage(&msg, &newRoom)
+				continue
+			}
+
+			// Handle sending messages
+			if msg.MessageType == ocs.MessageComment {
+				b.handleSendingMessage(&msg, &newRoom)
+				continue
+			}
+
+		}
+	}()
+	return nil
+}
+
+func (b *Btalk) Send(msg config.Message) (string, error) {
+	r := b.getRoom(msg.Channel)
+	if r == nil {
+		b.Log.Errorf("Could not find room for %v", msg.Channel)
+		return "", nil
+	}
+
+	// Standard Message Send
+	if msg.Event == "" {
+		// Handle sending files if they are included
+		err := b.handleSendingFile(&msg, r)
+		if err != nil {
+			b.Log.Errorf("Could not send files in message to room %v from %v: %v", msg.Channel, msg.Username, err)
+
+			return "", nil
+		}
+
+		sentMessage, err := r.room.SendMessage(msg.Username + msg.Text)
+		if err != nil {
+			b.Log.Errorf("Could not send message to room %v from %v: %v", msg.Channel, msg.Username, err)
+
+			return "", nil
+		}
+		return strconv.Itoa(sentMessage.ID), nil
+	}
+
+	// Message Deletion
+	if msg.Event == config.EventMsgDelete {
+		messageID, err := strconv.Atoi(msg.ID)
+		if err != nil {
+			return "", err
+		}
+		data, err := r.room.DeleteMessage(messageID)
+		if err != nil {
+			return "", err
+		}
+		return strconv.Itoa(data.ID), nil
+	}
+
+	// Message is not a type that is currently supported
+	return "", nil
+}
+
+func (b *Btalk) getRoom(token string) *Broom {
+	for _, r := range b.rooms {
+		if r.room.Token == token {
+			return &r
+		}
+	}
+	return nil
+}
+
+func (b *Btalk) handleFiles(mmsg *config.Message, message *ocs.TalkRoomMessageData) error {
+	for _, parameter := range message.MessageParameters {
+		if parameter.Type == ocs.ROSTypeFile {
+			// Get the file
+			file, err := b.user.DownloadFile(parameter.Path)
+			if err != nil {
+				return err
+			}
+
+			if mmsg.Extra == nil {
+				mmsg.Extra = make(map[string][]interface{})
+			}
+
+			mmsg.Extra["file"] = append(mmsg.Extra["file"], config.FileInfo{
+				Name:   parameter.Name,
+				Data:   file,
+				Size:   int64(len(*file)),
+				Avatar: false,
+			})
+		}
+	}
+
+	return nil
+}
+
+func (b *Btalk) handleSendingFile(msg *config.Message, r *Broom) error {
+	for _, f := range msg.Extra["file"] {
+		fi := f.(config.FileInfo)
+		if fi.URL == "" {
+			continue
+		}
+
+		message := msg.Username
+		if fi.Comment != "" {
+			message += fi.Comment + " "
+		}
+		message += fi.URL
+		_, err := r.room.SendMessage(message)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (b *Btalk) handleSendingMessage(msg *ocs.TalkRoomMessageData, r *Broom) {
+	remoteMessage := config.Message{
+		Text:     formatRichObjectString(msg.Message, msg.MessageParameters),
+		Channel:  r.room.Token,
+		Username: DisplayName(msg, b.guestSuffix()),
+		UserID:   msg.ActorID,
+		Account:  b.Account,
+	}
+	// It is possible for the ID to not be set on older versions of Talk so we only set it if
+	// the ID is not blank
+	if msg.ID != 0 {
+		remoteMessage.ID = strconv.Itoa(msg.ID)
+	}
+
+	// Handle Files
+	err := b.handleFiles(&remoteMessage, msg)
+	if err != nil {
+		b.Log.Errorf("Error handling file: %#v", msg)
+
+		return
+	}
+
+	b.Log.Debugf("<= Message is %#v", remoteMessage)
+	b.Remote <- remoteMessage
+}
+
+func (b *Btalk) handleDeletingMessage(msg *ocs.TalkRoomMessageData, r *Broom) {
+	remoteMessage := config.Message{
+		Event:   config.EventMsgDelete,
+		Text:    config.EventMsgDelete,
+		Channel: r.room.Token,
+		ID:      strconv.Itoa(msg.Parent.ID),
+		Account: b.Account,
+	}
+	b.Log.Debugf("<= Message being deleted is %#v", remoteMessage)
+	b.Remote <- remoteMessage
+}
+
+func (b *Btalk) guestSuffix() string {
+	guestSuffix := " (Guest)"
+	if b.IsKeySet("GuestSuffix") {
+		guestSuffix = b.GetString("GuestSuffix")
+	}
+
+	return guestSuffix
+}
+
+// Spec: https://github.com/nextcloud/server/issues/1706#issue-182308785
+func formatRichObjectString(message string, parameters map[string]ocs.RichObjectString) string {
+	for id, parameter := range parameters {
+		text := parameter.Name
+
+		switch parameter.Type {
+		case ocs.ROSTypeUser, ocs.ROSTypeGroup:
+			text = "@" + text
+		case ocs.ROSTypeFile:
+			if parameter.Link != "" {
+				text = parameter.Name
+			}
+		}
+
+		message = strings.ReplaceAll(message, "{"+id+"}", text)
+	}
+
+	return message
+}
+
+func DisplayName(msg *ocs.TalkRoomMessageData, suffix string) string {
+	if msg.ActorType == ocs.ActorGuest {
+		if msg.ActorDisplayName == "" {
+			return "Guest"
+		}
+
+		return msg.ActorDisplayName + suffix
+	}
+
+	return msg.ActorDisplayName
+}

bridge/rocketchat/handlers.go

@@ -1,7 +1,11 @@
 package brocketchat
 
 import (
+	"fmt"
+
 	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/42wim/matterbridge/bridge/helper"
+	"github.com/matterbridge/Rocket.Chat.Go.SDK/models"
 )
 
 func (b *Brocketchat) handleRocket() {
@@ -38,8 +42,26 @@ func (b *Brocketchat) handleRocketHook(messages chan *config.Message) {
 	}
 }
 
+func (b *Brocketchat) handleStatusEvent(ev models.Message, rmsg *config.Message) bool {
+	switch ev.Type {
+	case "":
+		// this is a normal message, no processing needed
+		// return true so the message is not dropped
+		return true
+	case sUserJoined, sUserLeft:
+		rmsg.Event = config.EventJoinLeave
+		return true
+	case sRoomChangedTopic:
+		rmsg.Event = config.EventTopicChange
+		return true
+	}
+	b.Log.Debugf("Dropping message with unknown type: %s", ev.Type)
+	return false
+}
+
 func (b *Brocketchat) handleRocketClient(messages chan *config.Message) {
 	for message := range b.messageChan {
+		message := message
 		// skip messages with same ID, apparently messages get duplicated for an unknown reason
 		if _, ok := b.cache.Get(message.ID); ok {
 			continue
@@ -58,11 +80,51 @@ func (b *Brocketchat) handleRocketClient(messages chan *config.Message) {
 			Account:  b.Account,
 			UserID:   message.User.ID,
 			ID:       message.ID,
+			Extra:    make(map[string][]interface{}),
+		}
+
+		b.handleAttachments(&message, rmsg)
+
+		// handleStatusEvent returns false if the message should be dropped
+		// in that case it is probably some modification to the channel we do not want to relay
+		if b.handleStatusEvent(m, rmsg) {
+			messages <- rmsg
 		}
-		messages <- rmsg
 	}
 }
 
+func (b *Brocketchat) handleAttachments(message *models.Message, rmsg *config.Message) {
+	if rmsg.Text == "" {
+		for _, attachment := range message.Attachments {
+			if attachment.Title != "" {
+				rmsg.Text = attachment.Title + "\n"
+			}
+			if attachment.Title != "" && attachment.Text != "" {
+				rmsg.Text += "\n"
+			}
+			if attachment.Text != "" {
+				rmsg.Text += attachment.Text
+			}
+		}
+	}
+
+	for i := range message.Attachments {
+		if err := b.handleDownloadFile(rmsg, &message.Attachments[i]); err != nil {
+			b.Log.Errorf("Could not download incoming file: %#v", err)
+		}
+	}
+}
+
+func (b *Brocketchat) handleDownloadFile(rmsg *config.Message, file *models.Attachment) error {
+	downloadURL := b.GetString("server") + file.TitleLink
+	data, err := helper.DownloadFileAuthRocket(downloadURL, b.user.Token, b.user.ID)
+	if err != nil {
+		return fmt.Errorf("download %s failed %#v", downloadURL, err)
+	}
+	helper.HandleDownloadData(b.Log, rmsg, file.Title, rmsg.Text, downloadURL, data, b.General)
+	return nil
+}
+
 func (b *Brocketchat) handleUploadFile(msg *config.Message) error {
 	for _, f := range msg.Extra["file"] {
 		fi := f.(config.FileInfo)

bridge/rocketchat/rocketchat.go

@@ -29,6 +29,12 @@ type Brocketchat struct {
 	sync.RWMutex
 }
 
+const (
+	sUserJoined       = "uj"
+	sUserLeft         = "ul"
+	sRoomChangedTopic = "room_changed_topic"
+)
+
 func New(cfg *bridge.Config) bridge.Bridger {
 	newCache, err := lru.New(100)
 	if err != nil {

bridge/slack/handlers.go

@@ -1,6 +1,7 @@
 package bslack
 
 import (
+	"errors"
 	"fmt"
 	"html"
 	"time"
@@ -10,9 +11,12 @@ import (
 	"github.com/slack-go/slack"
 )
 
+// ErrEventIgnored is for events that should be ignored
+var ErrEventIgnored = errors.New("this event message should ignored")
+
 func (b *Bslack) handleSlack() {
 	messages := make(chan *config.Message)
-	if b.GetString(incomingWebhookConfig) != "" {
+	if b.GetString(incomingWebhookConfig) != "" && b.GetString(tokenConfig) == "" {
 		b.Log.Debugf("Choosing webhooks based receiving")
 		go b.handleMatterHook(messages)
 	} else {
@@ -53,7 +57,9 @@ func (b *Bslack) handleSlackClient(messages chan *config.Message) {
 				continue
 			}
 			rmsg, err := b.handleTypingEvent(ev)
-			if err != nil {
+			if err == ErrEventIgnored {
+				continue
+			} else if err != nil {
 				b.Log.Errorf("%#v", err)
 				continue
 			}
@@ -131,12 +137,6 @@ func (b *Bslack) skipMessageEvent(ev *slack.MessageEvent) bool {
 		hasOurCallbackID = ok && block.BlockID == "matterbridge_"+b.uuid
 	}
 
-	// Skip any messages that we made ourselves or from 'slackbot' (see #527).
-	if ev.Username == sSlackBotUser ||
-		(b.rtm != nil && ev.Username == b.si.User.Name) || hasOurCallbackID {
-		return true
-	}
-
 	if ev.SubMessage != nil {
 		// It seems ev.SubMessage.Edited == nil when slack unfurls.
 		// Do not forward these messages. See Github issue #266.
@@ -149,6 +149,16 @@ func (b *Bslack) skipMessageEvent(ev *slack.MessageEvent) bool {
 		if ev.SubType == "message_replied" && ev.Hidden {
 			return true
 		}
+		if len(ev.SubMessage.Blocks.BlockSet) == 1 {
+			block, ok := ev.SubMessage.Blocks.BlockSet[0].(*slack.SectionBlock)
+			hasOurCallbackID = ok && block.BlockID == "matterbridge_"+b.uuid
+		}
+	}
+
+	// Skip any messages that we made ourselves or from 'slackbot' (see #527).
+	if ev.Username == sSlackBotUser ||
+		(b.rtm != nil && ev.Username == b.si.User.Name) || hasOurCallbackID {
+		return true
 	}
 
 	if len(ev.Files) > 0 {
@@ -276,6 +286,9 @@ func (b *Bslack) handleAttachments(ev *slack.MessageEvent, rmsg *config.Message)
 }
 
 func (b *Bslack) handleTypingEvent(ev *slack.UserTypingEvent) (*config.Message, error) {
+	if ev.User == b.si.User.ID {
+		return nil, ErrEventIgnored
+	}
 	channelInfo, err := b.channels.getChannelByID(ev.Channel)
 	if err != nil {
 		return nil, err

bridge/slack/slack.go

@@ -64,6 +64,7 @@ const (
 	editSuffixConfig      = "EditSuffix"
 	iconURLConfig         = "iconurl"
 	noSendJoinConfig      = "nosendjoinpart"
+	messageLength         = 3000
 )
 
 func New(cfg *bridge.Config) bridge.Bridger {
@@ -155,7 +156,7 @@ func (b *Bslack) JoinChannel(channel config.ChannelInfo) error {
 
 	// try to join a channel when in legacy
 	if b.legacy {
-		_, err := b.sc.JoinChannel(channel.Name)
+		_, _, _, err := b.sc.JoinConversation(channel.Name)
 		if err != nil {
 			switch err.Error() {
 			case "name_taken", "restricted_action":
@@ -194,6 +195,7 @@ func (b *Bslack) Send(msg config.Message) (string, error) {
 		b.Log.Debugf("=> Receiving %#v", msg)
 	}
 
+	msg.Text = helper.ClipMessage(msg.Text, messageLength, b.GetString("MessageClipped"))
 	msg.Text = b.replaceCodeFence(msg.Text)
 
 	// Make a action /me of the message
@@ -202,7 +204,7 @@ func (b *Bslack) Send(msg config.Message) (string, error) {
 	}
 
 	// Use webhook to send the message
-	if b.GetString(outgoingWebhookConfig) != "" {
+	if b.GetString(outgoingWebhookConfig) != "" && b.GetString(tokenConfig) == "" {
 		return "", b.sendWebhook(msg)
 	}
 	return b.sendRTM(msg)
@@ -297,7 +299,7 @@ func (b *Bslack) sendRTM(msg config.Message) (string, error) {
 	}
 
 	// Handle prefix hint for unthreaded messages.
-	if msg.ParentID == "msg-parent-not-found" {
+	if msg.ParentNotFound() {
 		msg.ParentID = ""
 		msg.Text = fmt.Sprintf("[thread]: %s", msg.Text)
 	}

bridge/slack/users_channels.go

@@ -283,7 +283,7 @@ func (b *channels) populateChannels(wait bool) {
 	// We only retrieve public and private channels, not IMs
 	// and MPIMs as those do not have a channel name.
 	queryParams := &slack.GetConversationsParameters{
-		ExcludeArchived: "true",
+		ExcludeArchived: true,
 		Types:           []string{"public_channel,private_channel"},
 	}
 	for {

bridge/telegram/handlers.go

@@ -2,7 +2,7 @@ package btelegram
 
 import (
 	"html"
-	"regexp"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"unicode/utf16"
@@ -39,22 +39,32 @@ func (b *Btelegram) handleGroups(rmsg *config.Message, message *tgbotapi.Message
 
 // handleForwarded handles forwarded messages
 func (b *Btelegram) handleForwarded(rmsg *config.Message, message *tgbotapi.Message) {
-	if message.ForwardFrom != nil {
-		usernameForward := ""
-		if b.GetBool("UseFirstName") {
+	if message.ForwardDate == 0 {
+		return
+	}
+
+	if message.ForwardFrom == nil {
+		rmsg.Text = "Forwarded from " + unknownUser + ": " + rmsg.Text
+		return
+	}
+
+	usernameForward := ""
+	if b.GetBool("UseFirstName") {
+		usernameForward = message.ForwardFrom.FirstName
+	}
+
+	if usernameForward == "" {
+		usernameForward = message.ForwardFrom.UserName
+		if usernameForward == "" {
 			usernameForward = message.ForwardFrom.FirstName
 		}
-		if usernameForward == "" {
-			usernameForward = message.ForwardFrom.UserName
-			if usernameForward == "" {
-				usernameForward = message.ForwardFrom.FirstName
-			}
-		}
-		if usernameForward == "" {
-			usernameForward = unknownUser
-		}
-		rmsg.Text = "Forwarded from " + usernameForward + ": " + rmsg.Text
 	}
+
+	if usernameForward == "" {
+		usernameForward = unknownUser
+	}
+
+	rmsg.Text = "Forwarded from " + usernameForward + ": " + rmsg.Text
 }
 
 // handleQuoting handles quoting of previous messages
@@ -171,13 +181,15 @@ func (b *Btelegram) handleRecv(updates <-chan tgbotapi.Update) {
 // sends a EVENT_AVATAR_DOWNLOAD message to the gateway if successful.
 // logs an error message if it fails
 func (b *Btelegram) handleDownloadAvatar(userid int, channel string) {
-	rmsg := config.Message{Username: "system",
-		Text:    "avatar",
-		Channel: channel,
-		Account: b.Account,
-		UserID:  strconv.Itoa(userid),
-		Event:   config.EventAvatarDownload,
-		Extra:   make(map[string][]interface{})}
+	rmsg := config.Message{
+		Username: "system",
+		Text:     "avatar",
+		Channel:  channel,
+		Account:  b.Account,
+		UserID:   strconv.Itoa(userid),
+		Event:    config.EventAvatarDownload,
+		Extra:    make(map[string][]interface{}),
+	}
 
 	if _, ok := b.avatarMap[strconv.Itoa(userid)]; !ok {
 		photos, err := b.c.GetUserProfilePhotos(tgbotapi.UserProfilePhotosConfig{UserID: userid, Limit: 1})
@@ -207,6 +219,46 @@ func (b *Btelegram) handleDownloadAvatar(userid int, channel string) {
 	}
 }
 
+func (b *Btelegram) maybeConvertTgs(name *string, data *[]byte) {
+	var format string
+	switch b.GetString("MediaConvertTgs") {
+	case FormatWebp:
+		b.Log.Debugf("Tgs to WebP conversion enabled, converting %v", name)
+		format = FormatWebp
+	case FormatPng:
+		// The WebP to PNG converter can't handle animated webp files yet,
+		// and I'm not going to write a path for x/image/webp.
+		// The error message would be:
+		//     conversion failed: webp: non-Alpha VP8X is not implemented
+		// So instead, we tell lottie to directly go to PNG.
+		b.Log.Debugf("Tgs to PNG conversion enabled, converting %v", name)
+		format = FormatPng
+	default:
+		// Otherwise, no conversion was requested. Trying to run the usual webp
+		// converter would fail, because '.tgs.webp' is actually a gzipped JSON
+		// file, and has nothing to do with WebP.
+		return
+	}
+	err := helper.ConvertTgsToX(data, format, b.Log)
+	if err != nil {
+		b.Log.Errorf("conversion failed: %v", err)
+	} else {
+		*name = strings.Replace(*name, "tgs.webp", format, 1)
+	}
+}
+
+func (b *Btelegram) maybeConvertWebp(name *string, data *[]byte) {
+	if b.GetBool("MediaConvertWebPToPNG") {
+		b.Log.Debugf("WebP to PNG conversion enabled, converting %v", name)
+		err := helper.ConvertWebPToPNG(data)
+		if err != nil {
+			b.Log.Errorf("conversion failed: %v", err)
+		} else {
+			*name = strings.Replace(*name, ".webp", ".png", 1)
+		}
+	}
+}
+
 // handleDownloadFile handles file download
 func (b *Btelegram) handleDownload(rmsg *config.Message, message *tgbotapi.Message) error {
 	size := 0
@@ -254,15 +306,18 @@ func (b *Btelegram) handleDownload(rmsg *config.Message, message *tgbotapi.Messa
 	if err != nil {
 		return err
 	}
-	if strings.HasSuffix(name, ".webp") && b.GetBool("MediaConvertWebPToPNG") {
-		b.Log.Debugf("WebP to PNG conversion enabled, converting %s", name)
-		err := helper.ConvertWebPToPNG(data)
-		if err != nil {
-			b.Log.Errorf("conversion failed: %s", err)
-		} else {
-			name = strings.Replace(name, ".webp", ".png", 1)
-		}
+
+	if strings.HasSuffix(name, ".tgs.webp") {
+		b.maybeConvertTgs(&name, data)
+	} else if strings.HasSuffix(name, ".webp") {
+		b.maybeConvertWebp(&name, data)
 	}
+
+	// rename .oga to .ogg  https://github.com/42wim/matterbridge/issues/906#issuecomment-741793512
+	if strings.HasSuffix(name, ".oga") && message.Audio != nil {
+		name = strings.Replace(name, ".oga", ".ogg", 1)
+	}
+
 	helper.HandleDownloadData(b.Log, rmsg, name, message.Caption, "", data, b.General)
 	return nil
 }
@@ -312,6 +367,9 @@ func (b *Btelegram) handleEdit(msg *config.Message, chatid int64) (string, error
 	case "Markdown":
 		b.Log.Debug("Using mode markdown")
 		m.ParseMode = tgbotapi.ModeMarkdown
+	case MarkdownV2:
+		b.Log.Debug("Using mode MarkdownV2")
+		m.ParseMode = MarkdownV2
 	}
 	if strings.ToLower(b.GetString("MessageFormat")) == HTMLNick {
 		b.Log.Debug("Using mode HTML - nick only")
@@ -333,21 +391,32 @@ func (b *Btelegram) handleUploadFile(msg *config.Message, chatid int64) string {
 			Name:  fi.Name,
 			Bytes: *fi.Data,
 		}
-		re := regexp.MustCompile(".(jpg|png)$")
-		if re.MatchString(fi.Name) {
-			c = tgbotapi.NewPhotoUpload(chatid, file)
-		} else {
-			c = tgbotapi.NewDocumentUpload(chatid, file)
+		switch filepath.Ext(fi.Name) {
+		case ".jpg", ".jpe", ".png":
+			pc := tgbotapi.NewPhotoUpload(chatid, file)
+			pc.Caption, pc.ParseMode = TGGetParseMode(b, msg.Username, fi.Comment)
+			c = pc
+		case ".mp4", ".m4v":
+			vc := tgbotapi.NewVideoUpload(chatid, file)
+			vc.Caption, vc.ParseMode = TGGetParseMode(b, msg.Username, fi.Comment)
+			c = vc
+		case ".mp3", ".oga":
+			ac := tgbotapi.NewAudioUpload(chatid, file)
+			ac.Caption, ac.ParseMode = TGGetParseMode(b, msg.Username, fi.Comment)
+			c = ac
+		case ".ogg":
+			voc := tgbotapi.NewVoiceUpload(chatid, file)
+			voc.Caption, voc.ParseMode = TGGetParseMode(b, msg.Username, fi.Comment)
+			c = voc
+		default:
+			dc := tgbotapi.NewDocumentUpload(chatid, file)
+			dc.Caption, dc.ParseMode = TGGetParseMode(b, msg.Username, fi.Comment)
+			c = dc
 		}
 		_, err := b.c.Send(c)
 		if err != nil {
 			b.Log.Errorf("file upload failed: %#v", err)
 		}
-		if fi.Comment != "" {
-			if _, err := b.sendMessage(chatid, msg.Username, fi.Comment); err != nil {
-				b.Log.Errorf("posting file comment %s failed: %s", fi.Comment, err)
-			}
-		}
 	}
 	return ""
 }
@@ -357,7 +426,7 @@ func (b *Btelegram) handleQuote(message, quoteNick, quoteMessage string) string
 	if format == "" {
 		format = "{MESSAGE} (re @{QUOTENICK}: {QUOTEMESSAGE})"
 	}
-	quoteMessagelength := len(quoteMessage)
+	quoteMessagelength := len([]rune(quoteMessage))
 	if b.GetInt("QuoteLengthLimit") != 0 && quoteMessagelength >= b.GetInt("QuoteLengthLimit") {
 		runes := []rune(quoteMessage)
 		quoteMessage = string(runes[0:b.GetInt("QuoteLengthLimit")])

bridge/telegram/telegram.go

@@ -2,6 +2,7 @@ package btelegram
 
 import (
 	"html"
+	"log"
 	"strconv"
 	"strings"
 
@@ -15,6 +16,9 @@ const (
 	unknownUser = "unknown"
 	HTMLFormat  = "HTML"
 	HTMLNick    = "htmlnick"
+	MarkdownV2  = "MarkdownV2"
+	FormatPng   = "png"
+	FormatWebp  = "webp"
 )
 
 type Btelegram struct {
@@ -24,6 +28,16 @@ type Btelegram struct {
 }
 
 func New(cfg *bridge.Config) bridge.Bridger {
+	tgsConvertFormat := cfg.GetString("MediaConvertTgs")
+	if tgsConvertFormat != "" {
+		err := helper.CanConvertTgsToX()
+		if err != nil {
+			log.Fatalf("Telegram bridge configured to convert .tgs files to '%s', but lottie does not appear to work:\n%#v", tgsConvertFormat, err)
+		}
+		if tgsConvertFormat != FormatPng && tgsConvertFormat != FormatWebp {
+			log.Fatalf("Telegram bridge configured to convert .tgs files to '%s', but only '%s' and '%s' are supported.", FormatPng, FormatWebp, tgsConvertFormat)
+		}
+	}
 	return &Btelegram{Config: cfg, avatarMap: make(map[string]string)}
 }
 
@@ -55,6 +69,28 @@ func (b *Btelegram) JoinChannel(channel config.ChannelInfo) error {
 	return nil
 }
 
+func TGGetParseMode(b *Btelegram, username string, text string) (textout string, parsemode string) {
+	textout = username + text
+	if b.GetString("MessageFormat") == HTMLFormat {
+		b.Log.Debug("Using mode HTML")
+		parsemode = tgbotapi.ModeHTML
+	}
+	if b.GetString("MessageFormat") == "Markdown" {
+		b.Log.Debug("Using mode markdown")
+		parsemode = tgbotapi.ModeMarkdown
+	}
+	if b.GetString("MessageFormat") == MarkdownV2 {
+		b.Log.Debug("Using mode MarkdownV2")
+		parsemode = MarkdownV2
+	}
+	if strings.ToLower(b.GetString("MessageFormat")) == HTMLNick {
+		b.Log.Debug("Using mode HTML - nick only")
+		textout = username + html.EscapeString(text)
+		parsemode = tgbotapi.ModeHTML
+	}
+	return textout, parsemode
+}
+
 func (b *Btelegram) Send(msg config.Message) (string, error) {
 	b.Log.Debugf("=> Receiving %#v", msg)
 
@@ -117,20 +153,7 @@ func (b *Btelegram) getFileDirectURL(id string) string {
 
 func (b *Btelegram) sendMessage(chatid int64, username, text string) (string, error) {
 	m := tgbotapi.NewMessage(chatid, "")
-	m.Text = username + text
-	if b.GetString("MessageFormat") == HTMLFormat {
-		b.Log.Debug("Using mode HTML")
-		m.ParseMode = tgbotapi.ModeHTML
-	}
-	if b.GetString("MessageFormat") == "Markdown" {
-		b.Log.Debug("Using mode markdown")
-		m.ParseMode = tgbotapi.ModeMarkdown
-	}
-	if strings.ToLower(b.GetString("MessageFormat")) == HTMLNick {
-		b.Log.Debug("Using mode HTML - nick only")
-		m.Text = username + html.EscapeString(text)
-		m.ParseMode = tgbotapi.ModeHTML
-	}
+	m.Text, m.ParseMode = TGGetParseMode(b, username, text)
 
 	m.DisableWebPagePreview = b.GetBool("DisableWebPagePreview")
 

bridge/vk/vk.go

@@ -0,0 +1,327 @@
+package bvk
+
+import (
+	"bytes"
+	"context"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/42wim/matterbridge/bridge"
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/42wim/matterbridge/bridge/helper"
+
+	"github.com/SevereCloud/vksdk/v2/api"
+	"github.com/SevereCloud/vksdk/v2/events"
+	longpoll "github.com/SevereCloud/vksdk/v2/longpoll-bot"
+	"github.com/SevereCloud/vksdk/v2/object"
+)
+
+const (
+	audioMessage = "audio_message"
+	document     = "doc"
+	photo        = "photo"
+	video        = "video"
+	graffiti     = "graffiti"
+	sticker      = "sticker"
+	wall         = "wall"
+)
+
+type user struct {
+	lastname, firstname, avatar string
+}
+
+type Bvk struct {
+	c            *api.VK
+	usernamesMap map[int]user // cache of user names and avatar URLs
+	*bridge.Config
+}
+
+func New(cfg *bridge.Config) bridge.Bridger {
+	return &Bvk{usernamesMap: make(map[int]user), Config: cfg}
+}
+
+func (b *Bvk) Connect() error {
+	b.Log.Info("Connecting")
+	b.c = api.NewVK(b.GetString("Token"))
+	lp, err := longpoll.NewLongPoll(b.c, b.GetInt("GroupID"))
+	if err != nil {
+		b.Log.Debugf("%#v", err)
+
+		return err
+	}
+
+	lp.MessageNew(func(ctx context.Context, obj events.MessageNewObject) {
+		b.handleMessage(obj.Message, false)
+	})
+
+	b.Log.Info("Connection succeeded")
+
+	go func() {
+		err := lp.Run()
+		if err != nil {
+			b.Log.Fatal("Enable longpoll in group management")
+		}
+	}()
+
+	return nil
+}
+
+func (b *Bvk) Disconnect() error {
+	return nil
+}
+
+func (b *Bvk) JoinChannel(channel config.ChannelInfo) error {
+	return nil
+}
+
+func (b *Bvk) Send(msg config.Message) (string, error) {
+	b.Log.Debugf("=> Receiving %#v", msg)
+
+	peerID, err := strconv.Atoi(msg.Channel)
+	if err != nil {
+		return "", err
+	}
+
+	params := api.Params{}
+
+	text := msg.Username + msg.Text
+
+	if msg.Extra != nil {
+		if len(msg.Extra["file"]) > 0 {
+			// generate attachments string
+			attachment, urls := b.uploadFiles(msg.Extra, peerID)
+			params["attachment"] = attachment
+			text += urls
+		}
+	}
+
+	params["message"] = text
+
+	if msg.ID == "" {
+		// New message
+		params["random_id"] = time.Now().Unix()
+		params["peer_ids"] = msg.Channel
+
+		res, e := b.c.MessagesSendPeerIDs(params)
+		if e != nil {
+			return "", err
+		}
+
+		return strconv.Itoa(res[0].ConversationMessageID), nil
+	}
+	// Edit message
+	messageID, err := strconv.ParseInt(msg.ID, 10, 64)
+	if err != nil {
+		return "", err
+	}
+
+	params["peer_id"] = peerID
+	params["conversation_message_id"] = messageID
+
+	_, err = b.c.MessagesEdit(params)
+	if err != nil {
+		return "", err
+	}
+
+	return msg.ID, nil
+}
+
+func (b *Bvk) getUser(id int) user {
+	u, found := b.usernamesMap[id]
+	if !found {
+		b.Log.Debug("Fetching username for ", id)
+
+		if id >= 0 {
+			result, _ := b.c.UsersGet(api.Params{
+				"user_ids": id,
+				"fields":   "photo_200",
+			})
+
+			resUser := result[0]
+			u = user{lastname: resUser.LastName, firstname: resUser.FirstName, avatar: resUser.Photo200}
+			b.usernamesMap[id] = u
+		} else {
+			result, _ := b.c.GroupsGetByID(api.Params{
+				"group_id": id * -1,
+			})
+
+			resGroup := result[0]
+			u = user{lastname: resGroup.Name, avatar: resGroup.Photo200}
+		}
+	}
+
+	return u
+}
+
+func (b *Bvk) handleMessage(msg object.MessagesMessage, isFwd bool) {
+	b.Log.Debug("ChatID: ", msg.PeerID)
+	// fetch user info
+	u := b.getUser(msg.FromID)
+
+	rmsg := config.Message{
+		Text:     msg.Text,
+		Username: u.firstname + " " + u.lastname,
+		Avatar:   u.avatar,
+		Channel:  strconv.Itoa(msg.PeerID),
+		Account:  b.Account,
+		UserID:   strconv.Itoa(msg.FromID),
+		ID:       strconv.Itoa(msg.ConversationMessageID),
+		Extra:    make(map[string][]interface{}),
+	}
+
+	if msg.ReplyMessage != nil {
+		ur := b.getUser(msg.ReplyMessage.FromID)
+		rmsg.Text = "Re: " + ur.firstname + " " + ur.lastname + "\n" + rmsg.Text
+	}
+
+	if isFwd {
+		rmsg.Username = "Fwd: " + rmsg.Username
+	}
+
+	if len(msg.Attachments) > 0 {
+		urls, text := b.getFiles(msg.Attachments)
+
+		if text != "" {
+			rmsg.Text += "\n" + text
+		}
+
+		// download
+		b.downloadFiles(&rmsg, urls)
+	}
+
+	if len(msg.FwdMessages) > 0 {
+		rmsg.Text += strconv.Itoa(len(msg.FwdMessages)) + " forwarded messages"
+	}
+
+	b.Remote <- rmsg
+
+	if len(msg.FwdMessages) > 0 {
+		// recursive processing of forwarded messages
+		for _, m := range msg.FwdMessages {
+			m.PeerID = msg.PeerID
+			b.handleMessage(m, true)
+		}
+	}
+}
+
+func (b *Bvk) uploadFiles(extra map[string][]interface{}, peerID int) (string, string) {
+	var attachments []string
+	text := ""
+
+	for _, f := range extra["file"] {
+		fi := f.(config.FileInfo)
+
+		if fi.Comment != "" {
+			text += fi.Comment + "\n"
+		}
+		a, err := b.uploadFile(fi, peerID)
+		if err != nil {
+			b.Log.Error("File upload error ", fi.Name)
+		}
+
+		attachments = append(attachments, a)
+	}
+
+	return strings.Join(attachments, ","), text
+}
+
+func (b *Bvk) uploadFile(file config.FileInfo, peerID int) (string, error) {
+	r := bytes.NewReader(*file.Data)
+
+	photoRE := regexp.MustCompile(".(jpg|jpe|png)$")
+	if photoRE.MatchString(file.Name) {
+		p, err := b.c.UploadMessagesPhoto(peerID, r)
+		if err != nil {
+			return "", err
+		}
+
+		return photo + strconv.Itoa(p[0].OwnerID) + "_" + strconv.Itoa(p[0].ID), nil
+	}
+
+	var doctype string
+	if strings.Contains(file.Name, ".ogg") {
+		doctype = audioMessage
+	} else {
+		doctype = document
+	}
+
+	doc, err := b.c.UploadMessagesDoc(peerID, doctype, file.Name, "", r)
+	if err != nil {
+		return "", err
+	}
+
+	switch doc.Type {
+	case audioMessage:
+		return document + strconv.Itoa(doc.AudioMessage.OwnerID) + "_" + strconv.Itoa(doc.AudioMessage.ID), nil
+	case document:
+		return document + strconv.Itoa(doc.Doc.OwnerID) + "_" + strconv.Itoa(doc.Doc.ID), nil
+	}
+
+	return "", nil
+}
+
+func (b *Bvk) getFiles(attachments []object.MessagesMessageAttachment) ([]string, string) {
+	var urls []string
+	var text []string
+
+	for _, a := range attachments {
+		switch a.Type {
+		case photo:
+			var resolution float64 = 0
+			url := a.Photo.Sizes[0].URL
+			for _, size := range a.Photo.Sizes {
+				r := size.Height * size.Width
+				if resolution < r {
+					resolution = r
+					url = size.URL
+				}
+			}
+
+			urls = append(urls, url)
+
+		case document:
+			urls = append(urls, a.Doc.URL)
+
+		case graffiti:
+			urls = append(urls, a.Graffiti.URL)
+
+		case audioMessage:
+			urls = append(urls, a.AudioMessage.DocsDocPreviewAudioMessage.LinkOgg)
+
+		case sticker:
+			var resolution float64 = 0
+			url := a.Sticker.Images[0].URL
+			for _, size := range a.Sticker.Images {
+				r := size.Height * size.Width
+				if resolution < r {
+					resolution = r
+					url = size.URL
+				}
+			}
+			urls = append(urls, url+".png")
+		case video:
+			text = append(text, "https://vk.com/video"+strconv.Itoa(a.Video.OwnerID)+"_"+strconv.Itoa(a.Video.ID))
+
+		case wall:
+			text = append(text, "https://vk.com/wall"+strconv.Itoa(a.Wall.FromID)+"_"+strconv.Itoa(a.Wall.ID))
+
+		default:
+			text = append(text, "This attachment is not supported ("+a.Type+")")
+		}
+	}
+
+	return urls, strings.Join(text, "\n")
+}
+
+func (b *Bvk) downloadFiles(rmsg *config.Message, urls []string) {
+	for _, url := range urls {
+		data, err := helper.DownloadFile(url)
+		if err == nil {
+			urlPart := strings.Split(url, "/")
+			name := strings.Split(urlPart[len(urlPart)-1], "?")[0]
+			helper.HandleDownloadData(b.Log, rmsg, name, "", url, data, b.General)
+		}
+	}
+}

bridge/whatsapp/handlers.go

@@ -23,7 +23,9 @@ Check:
 // HandleError received from WhatsApp
 func (b *Bwhatsapp) HandleError(err error) {
 	// ignore received invalid data errors. https://github.com/42wim/matterbridge/issues/843
-	if strings.Contains(err.Error(), "error processing data: received invalid data") {
+	// ignore tag 174 errors. https://github.com/42wim/matterbridge/issues/1094
+	if strings.Contains(err.Error(), "error processing data: received invalid data") ||
+		strings.Contains(err.Error(), "invalid string with tag 174") {
 		return
 	}
 
@@ -46,15 +48,22 @@ func (b *Bwhatsapp) reconnect(err error) {
 		Max:    5 * time.Minute,
 		Jitter: true,
 	}
+
 	for {
 		d := bf.Duration()
+
 		b.Log.Errorf("Connection failed, underlying error: %v", err)
 		b.Log.Infof("Waiting %s...", d)
+
 		time.Sleep(d)
+
 		b.Log.Info("Reconnecting...")
+
 		err := b.conn.Restore()
 		if err == nil {
 			bf.Reset()
+			b.startedAt = uint64(time.Now().Unix())
+
 			return
 		}
 	}
@@ -62,7 +71,7 @@ func (b *Bwhatsapp) reconnect(err error) {
 
 // HandleTextMessage sent from WhatsApp, relay it to the brige
 func (b *Bwhatsapp) HandleTextMessage(message whatsapp.TextMessage) {
-	if message.Info.FromMe { // || !strings.Contains(strings.ToLower(message.Text), "@echo") {
+	if message.Info.FromMe {
 		return
 	}
 	// whatsapp sends last messages to show context , cut them
@@ -70,13 +79,13 @@ func (b *Bwhatsapp) HandleTextMessage(message whatsapp.TextMessage) {
 		return
 	}
 
-	messageTime := time.Unix(int64(message.Info.Timestamp), 0) // TODO check how behaves between timezones
 	groupJID := message.Info.RemoteJid
-
 	senderJID := message.Info.SenderJid
+
 	if len(senderJID) == 0 {
-		// TODO workaround till https://github.com/Rhymen/go-whatsapp/issues/86 resolved
-		senderJID = *message.Info.Source.Participant
+		if message.Info.Source != nil && message.Info.Source.Participant != nil {
+			senderJID = *message.Info.Source.Participant
+		}
 	}
 
 	// translate sender's JID to the nicest username we can get
@@ -97,108 +106,275 @@ func (b *Bwhatsapp) HandleTextMessage(message whatsapp.TextMessage) {
 			if mention == "" {
 				mention = "someone"
 			}
+
 			message.Text = strings.Replace(message.Text, "@"+numberAndSuffix[0], "@"+mention, 1)
 		}
 	}
 
-	b.Log.Debugf("<= Sending message from %s on %s to gateway", senderJID, b.Account)
 	rmsg := config.Message{
-		UserID:    senderJID,
-		Username:  senderName,
-		Text:      message.Text,
-		Timestamp: messageTime,
-		Channel:   groupJID,
-		Account:   b.Account,
-		Protocol:  b.Protocol,
-		Extra:     make(map[string][]interface{}),
+		UserID:   senderJID,
+		Username: senderName,
+		Text:     message.Text,
+		Channel:  groupJID,
+		Account:  b.Account,
+		Protocol: b.Protocol,
+		Extra:    make(map[string][]interface{}),
 		//	ParentID: TODO, // TODO handle thread replies  // map from Info.QuotedMessageID string
-		//	Event     string    `json:"event"`
-		//	Gateway   string  // will be added during message processing
-		ID: message.Info.Id}
+		ID: message.Info.Id,
+	}
 
 	if avatarURL, exists := b.userAvatars[senderJID]; exists {
 		rmsg.Avatar = avatarURL
 	}
 
+	b.Log.Debugf("<= Sending message from %s on %s to gateway", senderJID, b.Account)
 	b.Log.Debugf("<= Message is %#v", rmsg)
+
 	b.Remote <- rmsg
 }
 
 // HandleImageMessage sent from WhatsApp, relay it to the brige
 func (b *Bwhatsapp) HandleImageMessage(message whatsapp.ImageMessage) {
-	if message.Info.FromMe { // || !strings.Contains(strings.ToLower(message.Text), "@echo") {
+	if message.Info.FromMe || message.Info.Timestamp < b.startedAt {
 		return
 	}
 
-	// whatsapp sends last messages to show context , cut them
-	if message.Info.Timestamp < b.startedAt {
-		return
-	}
-
-	messageTime := time.Unix(int64(message.Info.Timestamp), 0) // TODO check how behaves between timezones
-	groupJID := message.Info.RemoteJid
-
 	senderJID := message.Info.SenderJid
-	// if len(senderJid) == 0 {
-	//   // TODO workaround till https://github.com/Rhymen/go-whatsapp/issues/86 resolved
-	//   senderJid = *message.Info.Source.Participant
-	// }
+	if len(message.Info.SenderJid) == 0 && message.Info.Source != nil && message.Info.Source.Participant != nil {
+		senderJID = *message.Info.Source.Participant
+	}
 
-	// translate sender's Jid to the nicest username we can get
-	senderName := b.getSenderName(senderJID)
+	senderName := b.getSenderName(message.Info.SenderJid)
 	if senderName == "" {
 		senderName = "Someone" // don't expose telephone number
 	}
 
-	b.Log.Debugf("<= Sending message from %s on %s to gateway", senderJID, b.Account)
 	rmsg := config.Message{
-		UserID:    senderJID,
-		Username:  senderName,
-		Timestamp: messageTime,
-		Channel:   groupJID,
-		Account:   b.Account,
-		Protocol:  b.Protocol,
-		Extra:     make(map[string][]interface{}),
-		//  ParentID: TODO,      // TODO handle thread replies  // map from Info.QuotedMessageID string
-		//  Event     string    `json:"event"`
-		//  Gateway   string     // will be added during message processing
-		ID: message.Info.Id}
+		UserID:   senderJID,
+		Username: senderName,
+		Channel:  message.Info.RemoteJid,
+		Account:  b.Account,
+		Protocol: b.Protocol,
+		Extra:    make(map[string][]interface{}),
+		ID:       message.Info.Id,
+	}
 
 	if avatarURL, exists := b.userAvatars[senderJID]; exists {
 		rmsg.Avatar = avatarURL
 	}
 
-	// Download and unencrypt content
-	data, err := message.Download()
+	fileExt, err := mime.ExtensionsByType(message.Type)
 	if err != nil {
-		b.Log.Errorf("%v", err)
+		b.Log.Errorf("Mimetype detection error: %s", err)
+
 		return
 	}
 
-	// Get file extension by mimetype
-	fileExt, err := mime.ExtensionsByType(message.Type)
-	if err != nil {
-		b.Log.Errorf("%v", err)
-		return
+	// rename .jfif to .jpg https://github.com/42wim/matterbridge/issues/1292
+	if fileExt[0] == ".jfif" {
+		fileExt[0] = ".jpg"
+	}
+
+	// rename .jpe to .jpg https://github.com/42wim/matterbridge/issues/1463
+	if fileExt[0] == ".jpe" {
+		fileExt[0] = ".jpg"
 	}
 
 	filename := fmt.Sprintf("%v%v", message.Info.Id, fileExt[0])
 
-	b.Log.Debugf("<= Image downloaded and unencrypted")
+	b.Log.Debugf("Trying to download %s with type %s", filename, message.Type)
+
+	data, err := message.Download()
+	if err != nil {
+		b.Log.Errorf("Download image failed: %s", err)
+
+		return
+	}
 
 	// Move file to bridge storage
 	helper.HandleDownloadData(b.Log, &rmsg, filename, message.Caption, "", &data, b.General)
 
-	b.Log.Debugf("<= Image Message is %#v", rmsg)
+	b.Log.Debugf("<= Sending message from %s on %s to gateway", senderJID, b.Account)
+	b.Log.Debugf("<= Message is %#v", rmsg)
+
 	b.Remote <- rmsg
 }
 
-//func (b *Bwhatsapp) HandleVideoMessage(message whatsapp.VideoMessage) {
-//	fmt.Println(message) // TODO implement
-//}
-//
-//func (b *Bwhatsapp) HandleJsonMessage(message string) {
-//	fmt.Println(message) // TODO implement
-//}
-// TODO HandleRawMessage
-// TODO HandleAudioMessage
+// HandleVideoMessage downloads video messages
+func (b *Bwhatsapp) HandleVideoMessage(message whatsapp.VideoMessage) {
+	if message.Info.FromMe || message.Info.Timestamp < b.startedAt {
+		return
+	}
+
+	senderJID := message.Info.SenderJid
+	if len(message.Info.SenderJid) == 0 && message.Info.Source != nil && message.Info.Source.Participant != nil {
+		senderJID = *message.Info.Source.Participant
+	}
+
+	senderName := b.getSenderName(message.Info.SenderJid)
+	if senderName == "" {
+		senderName = "Someone" // don't expose telephone number
+	}
+
+	rmsg := config.Message{
+		UserID:   senderJID,
+		Username: senderName,
+		Channel:  message.Info.RemoteJid,
+		Account:  b.Account,
+		Protocol: b.Protocol,
+		Extra:    make(map[string][]interface{}),
+		ID:       message.Info.Id,
+	}
+
+	if avatarURL, exists := b.userAvatars[senderJID]; exists {
+		rmsg.Avatar = avatarURL
+	}
+
+	fileExt, err := mime.ExtensionsByType(message.Type)
+	if err != nil {
+		b.Log.Errorf("Mimetype detection error: %s", err)
+
+		return
+	}
+
+	if len(fileExt) == 0 {
+		fileExt = append(fileExt, ".mp4")
+	}
+
+	filename := fmt.Sprintf("%v%v", message.Info.Id, fileExt[0])
+
+	b.Log.Debugf("Trying to download %s with size %#v and type %s", filename, message.Length, message.Type)
+
+	data, err := message.Download()
+	if err != nil {
+		b.Log.Errorf("Download video failed: %s", err)
+
+		return
+	}
+
+	// Move file to bridge storage
+	helper.HandleDownloadData(b.Log, &rmsg, filename, message.Caption, "", &data, b.General)
+
+	b.Log.Debugf("<= Sending message from %s on %s to gateway", senderJID, b.Account)
+	b.Log.Debugf("<= Message is %#v", rmsg)
+
+	b.Remote <- rmsg
+}
+
+// HandleAudioMessage downloads audio messages
+func (b *Bwhatsapp) HandleAudioMessage(message whatsapp.AudioMessage) {
+	if message.Info.FromMe || message.Info.Timestamp < b.startedAt {
+		return
+	}
+
+	senderJID := message.Info.SenderJid
+	if len(message.Info.SenderJid) == 0 && message.Info.Source != nil && message.Info.Source.Participant != nil {
+		senderJID = *message.Info.Source.Participant
+	}
+
+	senderName := b.getSenderName(message.Info.SenderJid)
+	if senderName == "" {
+		senderName = "Someone" // don't expose telephone number
+	}
+
+	rmsg := config.Message{
+		UserID:   senderJID,
+		Username: senderName,
+		Channel:  message.Info.RemoteJid,
+		Account:  b.Account,
+		Protocol: b.Protocol,
+		Extra:    make(map[string][]interface{}),
+		ID:       message.Info.Id,
+	}
+
+	if avatarURL, exists := b.userAvatars[senderJID]; exists {
+		rmsg.Avatar = avatarURL
+	}
+
+	fileExt, err := mime.ExtensionsByType(message.Type)
+	if err != nil {
+		b.Log.Errorf("Mimetype detection error: %s", err)
+
+		return
+	}
+
+	if len(fileExt) == 0 {
+		fileExt = append(fileExt, ".ogg")
+	}
+
+	filename := fmt.Sprintf("%v%v", message.Info.Id, fileExt[0])
+
+	b.Log.Debugf("Trying to download %s with size %#v and type %s", filename, message.Length, message.Type)
+
+	data, err := message.Download()
+	if err != nil {
+		b.Log.Errorf("Download audio failed: %s", err)
+
+		return
+	}
+
+	// Move file to bridge storage
+	helper.HandleDownloadData(b.Log, &rmsg, filename, "audio message", "", &data, b.General)
+
+	b.Log.Debugf("<= Sending message from %s on %s to gateway", senderJID, b.Account)
+	b.Log.Debugf("<= Message is %#v", rmsg)
+
+	b.Remote <- rmsg
+}
+
+// HandleDocumentMessage downloads documents
+func (b *Bwhatsapp) HandleDocumentMessage(message whatsapp.DocumentMessage) {
+	if message.Info.FromMe || message.Info.Timestamp < b.startedAt {
+		return
+	}
+
+	senderJID := message.Info.SenderJid
+	if len(message.Info.SenderJid) == 0 && message.Info.Source != nil && message.Info.Source.Participant != nil {
+		senderJID = *message.Info.Source.Participant
+	}
+
+	senderName := b.getSenderName(message.Info.SenderJid)
+	if senderName == "" {
+		senderName = "Someone" // don't expose telephone number
+	}
+
+	rmsg := config.Message{
+		UserID:   senderJID,
+		Username: senderName,
+		Channel:  message.Info.RemoteJid,
+		Account:  b.Account,
+		Protocol: b.Protocol,
+		Extra:    make(map[string][]interface{}),
+		ID:       message.Info.Id,
+	}
+
+	if avatarURL, exists := b.userAvatars[senderJID]; exists {
+		rmsg.Avatar = avatarURL
+	}
+
+	fileExt, err := mime.ExtensionsByType(message.Type)
+	if err != nil {
+		b.Log.Errorf("Mimetype detection error: %s", err)
+
+		return
+	}
+
+	filename := fmt.Sprintf("%v", message.FileName)
+
+	b.Log.Debugf("Trying to download %s with extension %s and type %s", filename, fileExt, message.Type)
+
+	data, err := message.Download()
+	if err != nil {
+		b.Log.Errorf("Download document message failed: %s", err)
+
+		return
+	}
+
+	// Move file to bridge storage
+	helper.HandleDownloadData(b.Log, &rmsg, filename, "document", "", &data, b.General)
+
+	b.Log.Debugf("<= Sending message from %s on %s to gateway", senderJID, b.Account)
+	b.Log.Debugf("<= Message is %#v", rmsg)
+
+	b.Remote <- rmsg
+}

bridge/whatsapp/helpers.go

@@ -6,22 +6,24 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"strings"
 
 	qrcodeTerminal "github.com/Baozisoftware/qrcode-terminal-go"
 	"github.com/Rhymen/go-whatsapp"
 )
 
 type ProfilePicInfo struct {
-	URL string `json:"eurl"`
-	Tag string `json:"tag"`
-
-	Status int16 `json:"status"`
+	URL    string `json:"eurl"`
+	Tag    string `json:"tag"`
+	Status int16  `json:"status"`
 }
 
 func qrFromTerminal(invert bool) chan string {
 	qr := make(chan string)
+
 	go func() {
 		terminal := qrcodeTerminal.New()
+
 		if invert {
 			terminal = qrcodeTerminal.New2(qrcodeTerminal.ConsoleColors.BrightWhite, qrcodeTerminal.ConsoleColors.BrightBlack, qrcodeTerminal.QRCodeRecoveryLevels.Medium)
 		}
@@ -44,13 +46,12 @@ func (b *Bwhatsapp) readSession() (whatsapp.Session, error) {
 	if err != nil {
 		return session, err
 	}
+
 	defer file.Close()
+
 	decoder := gob.NewDecoder(file)
-	err = decoder.Decode(&session)
-	if err != nil {
-		return session, err
-	}
-	return session, nil
+
+	return session, decoder.Decode(&session)
 }
 
 func (b *Bwhatsapp) writeSession(session whatsapp.Session) error {
@@ -65,11 +66,31 @@ func (b *Bwhatsapp) writeSession(session whatsapp.Session) error {
 	if err != nil {
 		return err
 	}
-	defer file.Close()
-	encoder := gob.NewEncoder(file)
-	err = encoder.Encode(session)
 
-	return err
+	defer file.Close()
+
+	encoder := gob.NewEncoder(file)
+
+	return encoder.Encode(session)
+}
+
+func (b *Bwhatsapp) restoreSession() (*whatsapp.Session, error) {
+	session, err := b.readSession()
+	if err != nil {
+		b.Log.Warn(err.Error())
+	}
+
+	b.Log.Debugln("Restoring WhatsApp session..")
+
+	session, err = b.conn.RestoreWithSession(session)
+	if err != nil {
+		// restore session connection timed out (I couldn't get over it without logging in again)
+		return nil, errors.New("failed to restore session: " + err.Error())
+	}
+
+	b.Log.Debugln("Session restored successfully!")
+
+	return &session, nil
 }
 
 func (b *Bwhatsapp) getSenderName(senderJid string) string {
@@ -80,8 +101,33 @@ func (b *Bwhatsapp) getSenderName(senderJid string) string {
 		// if user is not in phone contacts
 		// it is the most obvious scenario unless you sync your phone contacts with some remote updated source
 		// users can change it in their WhatsApp settings -> profile -> click on Avatar
-		return sender.Notify
+		if sender.Notify != "" {
+			return sender.Notify
+		}
+
+		if sender.Short != "" {
+			return sender.Short
+		}
 	}
+
+	// try to reload this contact
+	_, err := b.conn.Contacts()
+	if err != nil {
+		b.Log.Errorf("error on update of contacts: %v", err)
+	}
+
+	if contact, exists := b.conn.Store.Contacts[senderJid]; exists {
+		// Add it to the user map
+		b.users[senderJid] = contact
+
+		if contact.Name != "" {
+			return contact.Name
+		}
+		// if user is not in phone contacts
+		// same as above
+		return contact.Notify
+	}
+
 	return ""
 }
 
@@ -89,6 +135,7 @@ func (b *Bwhatsapp) getSenderNotify(senderJid string) string {
 	if sender, exists := b.users[senderJid]; exists {
 		return sender.Notify
 	}
+
 	return ""
 }
 
@@ -97,11 +144,20 @@ func (b *Bwhatsapp) GetProfilePicThumb(jid string) (*ProfilePicInfo, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to get avatar: %v", err)
 	}
+
 	content := <-data
 	info := &ProfilePicInfo{}
+
 	err = json.Unmarshal([]byte(content), info)
 	if err != nil {
 		return info, fmt.Errorf("failed to unmarshal avatar info: %v", err)
 	}
+
 	return info, nil
 }
+
+func isGroupJid(identifier string) bool {
+	return strings.HasSuffix(identifier, "@g.us") ||
+		strings.HasSuffix(identifier, "@temp") ||
+		strings.HasSuffix(identifier, "@broadcast")
+}

bridge/whatsapp/whatsapp.go

@@ -28,7 +28,6 @@ const (
 type Bwhatsapp struct {
 	*bridge.Config
 
-	// https://github.com/Rhymen/go-whatsapp/blob/c31092027237441cffba1b9cb148eadf7c83c3d2/session.go#L18-L21
 	session   *whatsapp.Session
 	conn      *whatsapp.Conn
 	startedAt uint64
@@ -40,6 +39,7 @@ type Bwhatsapp struct {
 // New Create a new WhatsApp bridge. This will be called for each [whatsapp.<server>] entry you have in the config file
 func New(cfg *bridge.Config) bridge.Bridger {
 	number := cfg.GetString(cfgNumber)
+
 	if number == "" {
 		cfg.Log.Fatalf("Missing configuration for WhatsApp bridge: Number")
 	}
@@ -50,21 +50,17 @@ func New(cfg *bridge.Config) bridge.Bridger {
 		users:       make(map[string]whatsapp.Contact),
 		userAvatars: make(map[string]string),
 	}
+
 	return b
 }
 
 // Connect to WhatsApp. Required implementation of the Bridger interface
-// https://github.com/42wim/matterbridge/blob/2cfd880cdb0df29771bf8f31df8d990ab897889d/bridge/bridge.go#L11-L16
 func (b *Bwhatsapp) Connect() error {
-	b.RLock() // TODO do we need locking for Whatsapp?
-	defer b.RUnlock()
-
 	number := b.GetString(cfgNumber)
 	if number == "" {
-		return errors.New("WhatsApp's telephone Number need to be configured")
+		return errors.New("whatsapp's telephone number need to be configured")
 	}
 
-	// https://github.com/Rhymen/go-whatsapp#creating-a-connection
 	b.Log.Debugln("Connecting to WhatsApp..")
 	conn, err := whatsapp.NewConn(20 * time.Second)
 	if err != nil {
@@ -77,35 +73,18 @@ func (b *Bwhatsapp) Connect() error {
 	b.Log.Debugln("WhatsApp connection successful")
 
 	// load existing session in order to keep it between restarts
-	if b.session == nil {
-		var session whatsapp.Session
-		session, err = b.readSession()
-
-		if err == nil {
-			b.Log.Debugln("Restoring WhatsApp session..")
-
-			// https://github.com/Rhymen/go-whatsapp#restore
-			session, err = b.conn.RestoreWithSession(session)
-			if err != nil {
-				// TODO return or continue to normal login?
-				// restore session connection timed out (I couldn't get over it without logging in again)
-				return errors.New("failed to restore session: " + err.Error())
-			}
-
-			b.session = &session
-			b.Log.Debugln("Session restored successfully!")
-		} else {
-			b.Log.Warn(err.Error())
-		}
+	b.session, err = b.restoreSession()
+	if err != nil {
+		b.Log.Warn(err.Error())
 	}
 
 	// login to a new session
 	if b.session == nil {
-		err = b.Login()
-		if err != nil {
+		if err = b.Login(); err != nil {
 			return err
 		}
 	}
+
 	b.startedAt = uint64(time.Now().Unix())
 
 	_, err = b.conn.Contacts()
@@ -113,6 +92,13 @@ func (b *Bwhatsapp) Connect() error {
 		return fmt.Errorf("error on update of contacts: %v", err)
 	}
 
+	// see https://github.com/Rhymen/go-whatsapp/issues/137#issuecomment-480316013
+	for len(b.conn.Store.Contacts) == 0 {
+		b.conn.Contacts() // nolint:errcheck
+
+		<-time.After(1 * time.Second)
+	}
+
 	// map all the users
 	for id, contact := range b.conn.Store.Contacts {
 		if !isGroupJid(id) && id != "status@broadcast" {
@@ -129,12 +115,13 @@ func (b *Bwhatsapp) Connect() error {
 			info, err := b.GetProfilePicThumb(jid)
 			if err != nil {
 				b.Log.Warnf("Could not get profile photo of %s: %v", jid, err)
-
 			} else {
-				// TODO any race conditions here?
+				b.Lock()
 				b.userAvatars[jid] = info.URL
+				b.Unlock()
 			}
 		}
+
 		b.Log.Debug("Finished getting avatars..")
 	}()
 
@@ -151,8 +138,10 @@ func (b *Bwhatsapp) Login() error {
 	session, err := b.conn.Login(qrChan)
 	if err != nil {
 		b.Log.Warnln("Failed to log in:", err)
+
 		return err
 	}
+
 	b.session = &session
 
 	b.Log.Infof("Logged into session: %#v", session)
@@ -163,74 +152,62 @@ func (b *Bwhatsapp) Login() error {
 		fmt.Fprintf(os.Stderr, "error saving session: %v\n", err)
 	}
 
-	// TODO change connection strings to configured ones longClientName:"github.com/rhymen/go-whatsapp", shortClientName:"go-whatsapp"}" prefix=whatsapp
-	// TODO get also a nice logo
-
-	// TODO notification about unplugged and dead battery
-	// conn.Info: Wid, Pushname, Connected, Battery, Plugged
-
 	return nil
 }
 
 // Disconnect is called while reconnecting to the bridge
-// TODO 42wim Documentation would be helpful on when reconnects happen and what should be done in this function
 // Required implementation of the Bridger interface
-// https://github.com/42wim/matterbridge/blob/2cfd880cdb0df29771bf8f31df8d990ab897889d/bridge/bridge.go#L11-L16
 func (b *Bwhatsapp) Disconnect() error {
 	// We could Logout, but that would close the session completely and would require a new QR code scan
 	// https://github.com/Rhymen/go-whatsapp/blob/c31092027237441cffba1b9cb148eadf7c83c3d2/session.go#L377-L381
 	return nil
 }
 
-func isGroupJid(identifier string) bool {
-	return strings.HasSuffix(identifier, "@g.us") || strings.HasSuffix(identifier, "@temp")
-}
-
 // JoinChannel Join a WhatsApp group specified in gateway config as channel='number-id@g.us' or channel='Channel name'
 // Required implementation of the Bridger interface
 // https://github.com/42wim/matterbridge/blob/2cfd880cdb0df29771bf8f31df8d990ab897889d/bridge/bridge.go#L11-L16
 func (b *Bwhatsapp) JoinChannel(channel config.ChannelInfo) error {
 	byJid := isGroupJid(channel.Name)
 
+	// see https://github.com/Rhymen/go-whatsapp/issues/137#issuecomment-480316013
+	for len(b.conn.Store.Contacts) == 0 {
+		b.conn.Contacts() // nolint:errcheck
+		<-time.After(1 * time.Second)
+	}
+
 	// verify if we are member of the given group
 	if byJid {
 		// channel.Name specifies static group jID, not the name
 		if _, exists := b.conn.Store.Contacts[channel.Name]; !exists {
 			return fmt.Errorf("account doesn't belong to group with jid %s", channel.Name)
 		}
-	} else {
-		// channel.Name specifies group name that might change, warn about it
-		var jids []string
-		for id, contact := range b.conn.Store.Contacts {
-			if isGroupJid(id) && contact.Name == channel.Name {
-				jids = append(jids, id)
-			}
-		}
 
-		switch len(jids) {
-		case 0:
-			// didn't match any group - print out possibilites
-			// TODO sort
-			// copy b;
-			//sort.Slice(people, func(i, j int) bool {
-			//	return people[i].Age > people[j].Age
-			//})
-			for id, contact := range b.conn.Store.Contacts {
-				if isGroupJid(id) {
-					b.Log.Infof("%s %s", contact.Jid, contact.Name)
-				}
-			}
-			return fmt.Errorf("please specify group's JID from the list above instead of the name '%s'", channel.Name)
+		return nil
+	}
 
-		case 1:
-			return fmt.Errorf("group name might change. Please configure gateway with channel=\"%v\" instead of channel=\"%v\"", jids[0], channel.Name)
-
-		default:
-			return fmt.Errorf("there is more than one group with name '%s'. Please specify one of JIDs as channel name: %v", channel.Name, jids)
+	// channel.Name specifies group name that might change, warn about it
+	var jids []string
+	for id, contact := range b.conn.Store.Contacts {
+		if isGroupJid(id) && contact.Name == channel.Name {
+			jids = append(jids, id)
 		}
 	}
 
-	return nil
+	switch len(jids) {
+	case 0:
+		// didn't match any group - print out possibilites
+		for id, contact := range b.conn.Store.Contacts {
+			if isGroupJid(id) {
+				b.Log.Infof("%s %s", contact.Jid, contact.Name)
+			}
+		}
+
+		return fmt.Errorf("please specify group's JID from the list above instead of the name '%s'", channel.Name)
+	case 1:
+		return fmt.Errorf("group name might change. Please configure gateway with channel=\"%v\" instead of channel=\"%v\"", jids[0], channel.Name)
+	default:
+		return fmt.Errorf("there is more than one group with name '%s'. Please specify one of JIDs as channel name: %v", channel.Name, jids)
+	}
 }
 
 // Post a document message from the bridge to WhatsApp
@@ -304,14 +281,12 @@ func (b *Bwhatsapp) Send(msg config.Message) (string, error) {
 		if msg.ID == "" {
 			// No message ID in case action is executed on a message sent before the bridge was started
 			// and then the bridge cache doesn't have this message ID mapped
-
-			// TODO 42wim Doesn't the app get clogged with a ton of IDs after some time of running?
-			// WhatsApp allows to set any ID so in that case we could use external IDs and don't do mapping
-			// but external IDs are not set
 			return "", nil
 		}
-		// TODO delete message on WhatsApp https://github.com/Rhymen/go-whatsapp/issues/100
-		return "", nil
+
+		_, err := b.conn.RevokeMessage(msg.Channel, msg.ID, true)
+
+		return "", err
 	}
 
 	// Edit message
@@ -319,7 +294,6 @@ func (b *Bwhatsapp) Send(msg config.Message) (string, error) {
 		b.Log.Debugf("updating message with id %s", msg.ID)
 
 		msg.Text += " (edited)"
-		// TODO handle edit as a message reply with updated text
 	}
 
 	// Handle Upload a file
@@ -349,16 +323,7 @@ func (b *Bwhatsapp) Send(msg config.Message) (string, error) {
 
 	b.Log.Debugf("=> Sending %#v", msg)
 
-	// create message ID
-	// TODO follow and act if https://github.com/Rhymen/go-whatsapp/issues/101 implemented
-	idBytes := make([]byte, 10)
-	if _, err := rand.Read(idBytes); err != nil {
-		b.Log.Warn(err.Error())
-	}
-	message.Info.Id = strings.ToUpper(hex.EncodeToString(idBytes))
-	_, err := b.conn.Send(message)
-
-	return message.Info.Id, err
+	return b.conn.Send(message)
 }
 
 // TODO do we want that? to allow login with QR code from a bridged channel? https://github.com/tulir/mautrix-whatsapp/blob/513eb18e2d59bada0dd515ee1abaaf38a3bfe3d5/commands.go#L76

bridge/xmpp/handler.go

@@ -0,0 +1,34 @@
+package bxmpp
+
+import (
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/42wim/matterbridge/bridge/helper"
+	"github.com/matterbridge/go-xmpp"
+)
+
+// handleDownloadAvatar downloads the avatar of userid from channel
+// sends a EVENT_AVATAR_DOWNLOAD message to the gateway if successful.
+// logs an error message if it fails
+func (b *Bxmpp) handleDownloadAvatar(avatar xmpp.AvatarData) {
+	rmsg := config.Message{
+		Username: "system",
+		Text:     "avatar",
+		Channel:  b.parseChannel(avatar.From),
+		Account:  b.Account,
+		UserID:   avatar.From,
+		Event:    config.EventAvatarDownload,
+		Extra:    make(map[string][]interface{}),
+	}
+	if _, ok := b.avatarMap[avatar.From]; !ok {
+		b.Log.Debugf("Avatar.From: %s", avatar.From)
+
+		err := helper.HandleDownloadSize(b.Log, &rmsg, avatar.From+".png", int64(len(avatar.Data)), b.General)
+		if err != nil {
+			b.Log.Error(err)
+			return
+		}
+		helper.HandleDownloadData(b.Log, &rmsg, avatar.From+".png", rmsg.Text, "", &avatar.Data, b.General)
+		b.Log.Debugf("Avatar download complete")
+		b.Remote <- rmsg
+	}
+}

bridge/xmpp/helpers.go

@@ -0,0 +1,30 @@
+package bxmpp
+
+import (
+	"regexp"
+
+	"github.com/42wim/matterbridge/bridge/config"
+)
+
+var pathRegex = regexp.MustCompile("[^a-zA-Z0-9]+")
+
+// GetAvatar constructs a URL for a given user-avatar if it is available in the cache.
+func getAvatar(av map[string]string, userid string, general *config.Protocol) string {
+	if hash, ok := av[userid]; ok {
+		// NOTE: This does not happen in bridge/helper/helper.go but messes up XMPP
+		id := pathRegex.ReplaceAllString(userid, "_")
+		return general.MediaServerDownload + "/" + hash + "/" + id + ".png"
+	}
+	return ""
+}
+
+func (b *Bxmpp) cacheAvatar(msg *config.Message) string {
+	fi := msg.Extra["file"][0].(config.FileInfo)
+	/* if we have a sha we have successfully uploaded the file to the media server,
+	so we can now cache the sha */
+	if fi.SHA != "" {
+		b.Log.Debugf("Added %s to %s in avatarMap", fi.SHA, msg.UserID)
+		b.avatarMap[msg.UserID] = fi.SHA
+	}
+	return ""
+}

bridge/xmpp/xmpp.go

@@ -1,8 +1,12 @@
 package bxmpp
 
 import (
+	"bytes"
 	"crypto/tls"
+	"encoding/json"
 	"fmt"
+	"net/http"
+	"net/url"
 	"strings"
 	"sync"
 	"time"
@@ -23,12 +27,17 @@ type Bxmpp struct {
 	xmppMap   map[string]string
 	connected bool
 	sync.RWMutex
+
+	avatarAvailability map[string]bool
+	avatarMap          map[string]string
 }
 
 func New(cfg *bridge.Config) bridge.Bridger {
 	return &Bxmpp{
-		Config:  cfg,
-		xmppMap: make(map[string]string),
+		Config:             cfg,
+		xmppMap:            make(map[string]string),
+		avatarAvailability: make(map[string]bool),
+		avatarMap:          make(map[string]string),
 	}
 }
 
@@ -67,17 +76,35 @@ func (b *Bxmpp) Send(msg config.Message) (string, error) {
 	if msg.Event == config.EventMsgDelete {
 		return "", nil
 	}
+
 	b.Log.Debugf("=> Receiving %#v", msg)
 
+	if msg.Event == config.EventAvatarDownload {
+		return b.cacheAvatar(&msg), nil
+	}
+
+	// Make a action /me of the message, prepend the username with it.
+	// https://xmpp.org/extensions/xep-0245.html
+	if msg.Event == config.EventUserAction {
+		msg.Username = "/me " + msg.Username
+	}
+
 	// Upload a file (in XMPP case send the upload URL because XMPP has no native upload support).
+	var err error
 	if msg.Extra != nil {
 		for _, rmsg := range helper.HandleExtra(&msg, b.General) {
 			b.Log.Debugf("=> Sending attachement message %#v", rmsg)
-			if _, err := b.xc.Send(xmpp.Chat{
-				Type:   "groupchat",
-				Remote: rmsg.Channel + "@" + b.GetString("Muc"),
-				Text:   rmsg.Username + rmsg.Text,
-			}); err != nil {
+			if b.GetString("WebhookURL") != "" {
+				err = b.postSlackCompatibleWebhook(msg)
+			} else {
+				_, err = b.xc.Send(xmpp.Chat{
+					Type:   "groupchat",
+					Remote: rmsg.Channel + "@" + b.GetString("Muc"),
+					Text:   rmsg.Username + rmsg.Text,
+				})
+			}
+
+			if err != nil {
 				b.Log.WithError(err).Error("Unable to send message with share URL.")
 			}
 		}
@@ -86,13 +113,24 @@ func (b *Bxmpp) Send(msg config.Message) (string, error) {
 		}
 	}
 
+	if b.GetString("WebhookURL") != "" {
+		b.Log.Debugf("Sending message using Webhook")
+		err := b.postSlackCompatibleWebhook(msg)
+		if err != nil {
+			b.Log.Errorf("Failed to send message using webhook: %s", err)
+			return "", err
+		}
+
+		return "", nil
+	}
+
+	// Post normal message.
 	var msgReplaceID string
 	msgID := xid.New().String()
 	if msg.ID != "" {
 		msgID = msg.ID
 		msgReplaceID = msg.ID
 	}
-	// Post normal message.
 	b.Log.Debugf("=> Sending message %#v", msg)
 	if _, err := b.xc.Send(xmpp.Chat{
 		Type:      "groupchat",
@@ -106,6 +144,30 @@ func (b *Bxmpp) Send(msg config.Message) (string, error) {
 	return msgID, nil
 }
 
+func (b *Bxmpp) postSlackCompatibleWebhook(msg config.Message) error {
+	type XMPPWebhook struct {
+		Username string `json:"username"`
+		Text     string `json:"text"`
+	}
+	webhookBody, err := json.Marshal(XMPPWebhook{
+		Username: msg.Username,
+		Text:     msg.Text,
+	})
+	if err != nil {
+		b.Log.Errorf("Failed to marshal webhook: %s", err)
+		return err
+	}
+
+	resp, err := http.Post(b.GetString("WebhookURL")+"/"+url.QueryEscape(msg.Channel), "application/json", bytes.NewReader(webhookBody))
+	if err != nil {
+		b.Log.Errorf("Failed to POST webhook: %s", err)
+		return err
+	}
+
+	resp.Body.Close()
+	return nil
+}
+
 func (b *Bxmpp) createXMPP() error {
 	if !strings.Contains(b.GetString("Jid"), "@") {
 		return fmt.Errorf("the Jid %s doesn't contain an @", b.GetString("Jid"))
@@ -114,20 +176,22 @@ func (b *Bxmpp) createXMPP() error {
 		ServerName:         strings.Split(b.GetString("Jid"), "@")[1],
 		InsecureSkipVerify: b.GetBool("SkipTLSVerify"), // nolint: gosec
 	}
+
+	xmpp.DebugWriter = b.Log.Writer()
+
 	options := xmpp.Options{
 		Host:                         b.GetString("Server"),
 		User:                         b.GetString("Jid"),
 		Password:                     b.GetString("Password"),
 		NoTLS:                        true,
-		StartTLS:                     true,
+		StartTLS:                     !b.GetBool("NoTLS"),
 		TLSConfig:                    tc,
 		Debug:                        b.GetBool("debug"),
-		Logger:                       b.Log.Writer(),
 		Session:                      true,
 		Status:                       "",
 		StatusMessage:                "",
 		Resource:                     "",
-		InsecureAllowUnencryptedAuth: false,
+		InsecureAllowUnencryptedAuth: b.GetBool("NoTLS"),
 	}
 	var err error
 	b.xc, err = options.NewClient()
@@ -228,6 +292,16 @@ func (b *Bxmpp) handleXMPP() error {
 					event = config.EventTopicChange
 				}
 
+				available, sok := b.avatarAvailability[v.Remote]
+				avatar := ""
+				if !sok {
+					b.Log.Debugf("Requesting avatar data")
+					b.avatarAvailability[v.Remote] = false
+					b.xc.AvatarRequestData(v.Remote)
+				} else if available {
+					avatar = getAvatar(b.avatarMap, v.Remote, b.General)
+				}
+
 				msgID := v.ID
 				if v.ReplaceID != "" {
 					msgID = v.ReplaceID
@@ -237,6 +311,7 @@ func (b *Bxmpp) handleXMPP() error {
 					Text:     v.Text,
 					Channel:  b.parseChannel(v.Remote),
 					Account:  b.Account,
+					Avatar:   avatar,
 					UserID:   v.Remote,
 					ID:       msgID,
 					Event:    event,
@@ -253,6 +328,10 @@ func (b *Bxmpp) handleXMPP() error {
 				b.Log.Debugf("<= Message is %#v", rmsg)
 				b.Remote <- rmsg
 			}
+		case xmpp.AvatarData:
+			b.handleDownloadAvatar(v)
+			b.avatarAvailability[v.From] = true
+			b.Log.Debugf("Avatar for %s is now available", v.From)
 		case xmpp.Presence:
 			// Do nothing.
 		}
@@ -345,6 +424,11 @@ func (b *Bxmpp) skipMessage(message xmpp.Chat) bool {
 		return true
 	}
 
+	// Ignore messages posted by our webhook
+	if b.GetString("WebhookURL") != "" && strings.Contains(message.ID, "webhookbot") {
+		return true
+	}
+
 	// skip delayed messages
 	return !message.Stamp.IsZero() && time.Since(message.Stamp).Minutes() > 5
 }

bridge/zulip/zulip.go

@@ -135,19 +135,25 @@ func (b *Bzulip) handleQueue() error {
 			if m.SenderEmail == b.GetString("login") {
 				continue
 			}
+
+			avatarURL := m.AvatarURL
+			if !strings.HasPrefix(avatarURL, "http") {
+				avatarURL = b.GetString("server") + avatarURL
+			}
+
 			rmsg := config.Message{
 				Username: m.SenderFullName,
 				Text:     m.Content,
 				Channel:  b.getChannel(m.StreamID) + "/topic:" + m.Subject,
 				Account:  b.Account,
 				UserID:   strconv.Itoa(m.SenderID),
-				Avatar:   m.AvatarURL,
+				Avatar:   avatarURL,
 			}
 			b.Log.Debugf("<= Sending message from %s on %s to gateway", rmsg.Username, b.Account)
 			b.Log.Debugf("<= Message is %#v", rmsg)
 			b.Remote <- rmsg
-			b.q.LastEventID = m.ID
 		}
+
 		time.Sleep(time.Second * 3)
 	}
 }

ci/bintray.sh

@@ -1,30 +0,0 @@
-#!/usr/bin/env bash
-set -u -e -x -o pipefail
-
-go version | grep go1.14 || exit
-
-VERSION=$(git describe --tags)
-mkdir ci/binaries
-GOOS=windows GOARCH=amd64 go build -ldflags "-s -w -X main.githash=$(git log --pretty=format:'%h' -n 1)" -o ci/binaries/matterbridge-$VERSION-windows-amd64.exe
-GOOS=linux GOARCH=amd64 go build -ldflags "-s -w -X main.githash=$(git log --pretty=format:'%h' -n 1)" -o ci/binaries/matterbridge-$VERSION-linux-amd64
-GOOS=linux GOARCH=arm go build -ldflags "-s -w -X main.githash=$(git log --pretty=format:'%h' -n 1)" -o ci/binaries/matterbridge-$VERSION-linux-arm
-GOOS=darwin GOARCH=amd64 go build -ldflags "-s -w -X main.githash=$(git log --pretty=format:'%h' -n 1)" -o ci/binaries/matterbridge-$VERSION-darwin-amd64
-cd ci
-cat > deploy.json <<EOF
-{
-    "package": {
-        "name": "Matterbridge",
-        "repo": "nightly",
-        "subject": "42wim"
-    },
-    "version": {
-        "name": "$VERSION"
-    },
-    "files":
-        [
-        {"includePattern": "ci/binaries/(.*)", "uploadPattern":"\$1"}
-        ],
-    "publish": true
-}
-EOF
-

ci/lint.sh

@@ -1,17 +0,0 @@
-#!/usr/bin/env bash
-set -u -e -x -o pipefail
-
-if [[ -n "${GOLANGCI_VERSION-}" ]]; then
-  # Retrieve the golangci-lint linter binary.
-  curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | bash -s -- -b ${GOPATH}/bin ${GOLANGCI_VERSION}
-fi
-
-# Run the linter.
-golangci-lint run
-
-# if [[ "${GO111MODULE-off}" == "on" ]]; then
-#   # If Go modules are active then check that dependencies are correctly maintained.
-#   go mod tidy
-#   go mod vendor
-#   git diff --exit-code --quiet || (echo "Please run 'go mod tidy' to clean up the 'go.mod' and 'go.sum' files."; false)
-# fi

ci/test.sh

@@ -1,17 +0,0 @@
-#!/usr/bin/env bash
-set -u -e -x -o pipefail
-
-if [[ -n "${REPORT_COVERAGE+cover}" ]]; then
-  # Retrieve and prepare CodeClimate's test coverage reporter.
-  curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
-  chmod +x ./cc-test-reporter
-  ./cc-test-reporter before-build
-fi
-
-# Run all the tests with the race detector and generate coverage.
-go test -v -race -coverprofile c.out ./...
-
-if [[ -n "${REPORT_COVERAGE+cover}" && "${TRAVIS_SECURE_ENV_VARS}" == "true" ]]; then
-  # Upload test coverage to CodeClimate.
-  ./cc-test-reporter after-build
-fi

gateway/bridgemap/api.go

@@ -0,0 +1,11 @@
+// +build !noapi
+
+package bridgemap
+
+import (
+	"github.com/42wim/matterbridge/bridge/api"
+)
+
+func init() {
+	FullMap["api"] = api.New
+}

gateway/bridgemap/bdiscord.go

@@ -0,0 +1,12 @@
+// +build !nodiscord
+
+package bridgemap
+
+import (
+	bdiscord "github.com/42wim/matterbridge/bridge/discord"
+)
+
+func init() {
+	FullMap["discord"] = bdiscord.New
+	UserTypingSupport["discord"] = struct{}{}
+}

gateway/bridgemap/bgitter.go

@@ -0,0 +1,11 @@
+// +build !nogitter
+
+package bridgemap
+
+import (
+	bgitter "github.com/42wim/matterbridge/bridge/gitter"
+)
+
+func init() {
+	FullMap["gitter"] = bgitter.New
+}

gateway/bridgemap/birc.go

@@ -0,0 +1,11 @@
+// +build !noirc
+
+package bridgemap
+
+import (
+	birc "github.com/42wim/matterbridge/bridge/irc"
+)
+
+func init() {
+	FullMap["irc"] = birc.New
+}

gateway/bridgemap/bkeybase.go

@@ -0,0 +1,11 @@
+// +build !nokeybase
+
+package bridgemap
+
+import (
+	bkeybase "github.com/42wim/matterbridge/bridge/keybase"
+)
+
+func init() {
+	FullMap["keybase"] = bkeybase.New
+}

gateway/bridgemap/bmatrix.go

@@ -0,0 +1,11 @@
+// +build !nomatrix
+
+package bridgemap
+
+import (
+	bmatrix "github.com/42wim/matterbridge/bridge/matrix"
+)
+
+func init() {
+	FullMap["matrix"] = bmatrix.New
+}

gateway/bridgemap/bmattermost.go

@@ -0,0 +1,11 @@
+// +build !nomattermost
+
+package bridgemap
+
+import (
+	bmattermost "github.com/42wim/matterbridge/bridge/mattermost"
+)
+
+func init() {
+	FullMap["mattermost"] = bmattermost.New
+}

gateway/bridgemap/bmsteams.go

@@ -0,0 +1,11 @@
+// +build !nomsteams
+
+package bridgemap
+
+import (
+	bmsteams "github.com/42wim/matterbridge/bridge/msteams"
+)
+
+func init() {
+	FullMap["msteams"] = bmsteams.New
+}

gateway/bridgemap/bmumble.go

@@ -0,0 +1,11 @@
+// +build !nomumble
+
+package bridgemap
+
+import (
+	bmumble "github.com/42wim/matterbridge/bridge/mumble"
+)
+
+func init() {
+	FullMap["mumble"] = bmumble.New
+}

gateway/bridgemap/bnctalk.go

@@ -0,0 +1,11 @@
+// +build !nonctalk
+
+package bridgemap
+
+import (
+	btalk "github.com/42wim/matterbridge/bridge/nctalk"
+)
+
+func init() {
+	FullMap["nctalk"] = btalk.New
+}

gateway/bridgemap/bridgemap.go

@@ -2,47 +2,9 @@ package bridgemap
 
 import (
 	"github.com/42wim/matterbridge/bridge"
-	"github.com/42wim/matterbridge/bridge/api"
-	bdiscord "github.com/42wim/matterbridge/bridge/discord"
-	bgitter "github.com/42wim/matterbridge/bridge/gitter"
-	birc "github.com/42wim/matterbridge/bridge/irc"
-	bkeybase "github.com/42wim/matterbridge/bridge/keybase"
-	bmatrix "github.com/42wim/matterbridge/bridge/matrix"
-	bmattermost "github.com/42wim/matterbridge/bridge/mattermost"
-	bmsteams "github.com/42wim/matterbridge/bridge/msteams"
-	brocketchat "github.com/42wim/matterbridge/bridge/rocketchat"
-	bslack "github.com/42wim/matterbridge/bridge/slack"
-	bsshchat "github.com/42wim/matterbridge/bridge/sshchat"
-	bsteam "github.com/42wim/matterbridge/bridge/steam"
-	btelegram "github.com/42wim/matterbridge/bridge/telegram"
-	bwhatsapp "github.com/42wim/matterbridge/bridge/whatsapp"
-	bxmpp "github.com/42wim/matterbridge/bridge/xmpp"
-	bzulip "github.com/42wim/matterbridge/bridge/zulip"
 )
 
 var (
-	FullMap = map[string]bridge.Factory{
-		"api":          api.New,
-		"discord":      bdiscord.New,
-		"gitter":       bgitter.New,
-		"irc":          birc.New,
-		"mattermost":   bmattermost.New,
-		"matrix":       bmatrix.New,
-		"rocketchat":   brocketchat.New,
-		"slack-legacy": bslack.NewLegacy,
-		"slack":        bslack.New,
-		"sshchat":      bsshchat.New,
-		"steam":        bsteam.New,
-		"telegram":     btelegram.New,
-		"whatsapp":     bwhatsapp.New,
-		"xmpp":         bxmpp.New,
-		"zulip":        bzulip.New,
-		"keybase":      bkeybase.New,
-		"msteams":      bmsteams.New,
-	}
-
-	UserTypingSupport = map[string]struct{}{
-		"slack":   {},
-		"discord": {},
-	}
+	FullMap           = map[string]bridge.Factory{}
+	UserTypingSupport = map[string]struct{}{}
 )

gateway/bridgemap/brocketchat.go

@@ -0,0 +1,11 @@
+// +build !norocketchat
+
+package bridgemap
+
+import (
+	brocketchat "github.com/42wim/matterbridge/bridge/rocketchat"
+)
+
+func init() {
+	FullMap["rocketchat"] = brocketchat.New
+}

gateway/bridgemap/bslack.go

@@ -0,0 +1,13 @@
+// +build !noslack
+
+package bridgemap
+
+import (
+	bslack "github.com/42wim/matterbridge/bridge/slack"
+)
+
+func init() {
+	FullMap["slack-legacy"] = bslack.NewLegacy
+	FullMap["slack"] = bslack.New
+	UserTypingSupport["slack"] = struct{}{}
+}

gateway/bridgemap/bsshchat.go

@@ -0,0 +1,11 @@
+// +build !nosshchat
+
+package bridgemap
+
+import (
+	bsshchat "github.com/42wim/matterbridge/bridge/sshchat"
+)
+
+func init() {
+	FullMap["sshchat"] = bsshchat.New
+}

gateway/bridgemap/bsteam.go

@@ -0,0 +1,11 @@
+// +build !nosteam
+
+package bridgemap
+
+import (
+	bsteam "github.com/42wim/matterbridge/bridge/steam"
+)
+
+func init() {
+	FullMap["steam"] = bsteam.New
+}

gateway/bridgemap/btelegram.go

@@ -0,0 +1,11 @@
+// +build !notelegram
+
+package bridgemap
+
+import (
+	btelegram "github.com/42wim/matterbridge/bridge/telegram"
+)
+
+func init() {
+	FullMap["telegram"] = btelegram.New
+}

gateway/bridgemap/bvk.go

@@ -0,0 +1,11 @@
+// +build !novk
+
+package bridgemap
+
+import (
+	bvk "github.com/42wim/matterbridge/bridge/vk"
+)
+
+func init() {
+	FullMap["vk"] = bvk.New
+}

gateway/bridgemap/bwhatsapp.go

@@ -0,0 +1,11 @@
+// +build !nowhatsapp
+
+package bridgemap
+
+import (
+	bwhatsapp "github.com/42wim/matterbridge/bridge/whatsapp"
+)
+
+func init() {
+	FullMap["whatsapp"] = bwhatsapp.New
+}

gateway/bridgemap/bxmpp.go

@@ -0,0 +1,11 @@
+// +build !noxmpp
+
+package bridgemap
+
+import (
+	bxmpp "github.com/42wim/matterbridge/bridge/xmpp"
+)
+
+func init() {
+	FullMap["xmpp"] = bxmpp.New
+}

gateway/bridgemap/bzulip.go

@@ -0,0 +1,11 @@
+// +build !nozulip
+
+package bridgemap
+
+import (
+	bzulip "github.com/42wim/matterbridge/bridge/zulip"
+)
+
+func init() {
+	FullMap["zulip"] = bzulip.New
+}

gateway/gateway.go

@@ -1,6 +1,7 @@
 package gateway
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
 	"regexp"
@@ -13,7 +14,7 @@ import (
 	"github.com/d5/tengo/v2"
 	"github.com/d5/tengo/v2/stdlib"
 	lru "github.com/hashicorp/golang-lru"
-	"github.com/matterbridge/emoji"
+	"github.com/kyokomi/emoji/v2"
 	"github.com/sirupsen/logrus"
 )
 
@@ -108,7 +109,7 @@ func (gw *Gateway) AddBridge(cfg *config.Bridge) error {
 func (gw *Gateway) checkConfig(cfg *config.Bridge) {
 	match := false
 	for _, key := range gw.Router.Config.Viper().AllKeys() {
-		if strings.HasPrefix(key, cfg.Account) {
+		if strings.HasPrefix(key, strings.ToLower(cfg.Account)) {
 			match = true
 			break
 		}
@@ -126,7 +127,7 @@ func (gw *Gateway) AddConfig(cfg *config.Gateway) error {
 		gw.logger.Errorf("mapChannels() failed: %s", err)
 	}
 	for _, br := range append(gw.MyConfig.In, append(gw.MyConfig.InOut, gw.MyConfig.Out...)...) {
-		br := br //scopelint
+		br := br // scopelint
 		err := gw.AddBridge(&br)
 		if err != nil {
 			return err
@@ -336,20 +337,21 @@ func (gw *Gateway) modifyUsername(msg *config.Message, dest *bridge.Bridge) stri
 			}
 			i++
 		}
-		nick = strings.Replace(nick, "{NOPINGNICK}", msg.Username[:i]+"​"+msg.Username[i:], -1)
+		nick = strings.ReplaceAll(nick, "{NOPINGNICK}", msg.Username[:i]+"\u200b"+msg.Username[i:])
 	}
 
-	nick = strings.Replace(nick, "{BRIDGE}", br.Name, -1)
-	nick = strings.Replace(nick, "{PROTOCOL}", br.Protocol, -1)
-	nick = strings.Replace(nick, "{GATEWAY}", gw.Name, -1)
-	nick = strings.Replace(nick, "{LABEL}", br.GetString("Label"), -1)
-	nick = strings.Replace(nick, "{NICK}", msg.Username, -1)
-	nick = strings.Replace(nick, "{CHANNEL}", msg.Channel, -1)
+	nick = strings.ReplaceAll(nick, "{BRIDGE}", br.Name)
+	nick = strings.ReplaceAll(nick, "{PROTOCOL}", br.Protocol)
+	nick = strings.ReplaceAll(nick, "{GATEWAY}", gw.Name)
+	nick = strings.ReplaceAll(nick, "{LABEL}", br.GetString("Label"))
+	nick = strings.ReplaceAll(nick, "{NICK}", msg.Username)
+	nick = strings.ReplaceAll(nick, "{USERID}", msg.UserID)
+	nick = strings.ReplaceAll(nick, "{CHANNEL}", msg.Channel)
 	tengoNick, err := gw.modifyUsernameTengo(msg, br)
 	if err != nil {
 		gw.logger.Errorf("modifyUsernameTengo error: %s", err)
 	}
-	nick = strings.Replace(nick, "{TENGO}", tengoNick, -1) //nolint:gocritic
+	nick = strings.ReplaceAll(nick, "{TENGO}", tengoNick)
 	return nick
 }
 
@@ -363,14 +365,28 @@ func (gw *Gateway) modifyAvatar(msg *config.Message, dest *bridge.Bridge) string
 }
 
 func (gw *Gateway) modifyMessage(msg *config.Message) {
-	if err := modifyMessageTengo(gw.BridgeValues().General.TengoModifyMessage, msg); err != nil {
+	if gw.BridgeValues().General.TengoModifyMessage != "" {
+		gw.logger.Warnf("General TengoModifyMessage=%s is deprecated and will be removed in v1.20.0, please move to Tengo InMessage=%s", gw.BridgeValues().General.TengoModifyMessage, gw.BridgeValues().General.TengoModifyMessage)
+	}
+
+	if err := modifyInMessageTengo(gw.BridgeValues().General.TengoModifyMessage, msg); err != nil {
 		gw.logger.Errorf("TengoModifyMessage failed: %s", err)
 	}
-	if err := modifyMessageTengo(gw.BridgeValues().Tengo.Message, msg); err != nil {
+
+	inMessage := gw.BridgeValues().Tengo.InMessage
+	if inMessage == "" {
+		inMessage = gw.BridgeValues().Tengo.Message
+		if inMessage != "" {
+			gw.logger.Warnf("Tengo Message=%s is deprecated and will be removed in v1.20.0, please move to Tengo InMessage=%s", inMessage, inMessage)
+		}
+	}
+
+	if err := modifyInMessageTengo(inMessage, msg); err != nil {
 		gw.logger.Errorf("Tengo.Message failed: %s", err)
 	}
 
 	// replace :emoji: to unicode
+	emoji.ReplacePadding = ""
 	msg.Text = emoji.Sprint(msg.Text)
 
 	br := gw.Bridges[msg.Account]
@@ -416,9 +432,15 @@ func (gw *Gateway) SendMessage(
 		}
 	}
 
+	// Only send irc notices to irc
+	if msg.Event == config.EventNoticeIRC && dest.Protocol != "irc" {
+		return "", nil
+	}
+
 	// Too noisy to log like other events
+	debugSendMessage := ""
 	if msg.Event != config.EventUserTyping {
-		gw.logger.Debugf("=> Sending %#v from %s (%s) to %s (%s)", msg, msg.Account, rmsg.Channel, dest.Account, channel.Name)
+		debugSendMessage = fmt.Sprintf("=> Sending %#v from %s (%s) to %s (%s)", msg, msg.Account, rmsg.Channel, dest.Account, channel.Name)
 	}
 
 	msg.Channel = channel.Name
@@ -438,22 +460,34 @@ func (gw *Gateway) SendMessage(
 	}
 
 	// if the parentID is still empty and we have a parentID set in the original message
-	// this means that we didn't find it in the cache so set it "msg-parent-not-found"
+	// this means that we didn't find it in the cache so set it to a "msg-parent-not-found" constant
 	if msg.ParentID == "" && rmsg.ParentID != "" {
-		msg.ParentID = "msg-parent-not-found"
+		msg.ParentID = config.ParentIDNotFound
 	}
 
-	err := gw.modifySendMessageTengo(rmsg, &msg, dest)
+	drop, err := gw.modifyOutMessageTengo(rmsg, &msg, dest)
 	if err != nil {
 		gw.logger.Errorf("modifySendMessageTengo: %s", err)
 	}
 
+	if drop {
+		gw.logger.Debugf("=> Tengo dropping %#v from %s (%s) to %s (%s)", msg, msg.Account, rmsg.Channel, dest.Account, channel.Name)
+		return "", nil
+	}
+
+	if debugSendMessage != "" {
+		gw.logger.Debug(debugSendMessage)
+	}
 	// if we are using mattermost plugin account, send messages to MattermostPlugin channel
 	// that can be picked up by the mattermost matterbridge plugin
 	if dest.Account == "mattermost.plugin" {
 		gw.Router.MattermostPlugin <- msg
 	}
 
+	defer func(t time.Time) {
+		gw.logger.Debugf("=> Send from %s (%s) to %s (%s) took %s", msg.Account, rmsg.Channel, dest.Account, channel.Name, time.Since(t))
+	}(time.Now())
+
 	mID, err := dest.Send(msg)
 	if err != nil {
 		return mID, err
@@ -463,7 +497,7 @@ func (gw *Gateway) SendMessage(
 	if mID != "" {
 		gw.logger.Debugf("mID %s: %s", dest.Account, mID)
 		return mID, nil
-		//brMsgIDs = append(brMsgIDs, &BrMsgID{dest, dest.Protocol + " " + mID, channel.ID})
+		// brMsgIDs = append(brMsgIDs, &BrMsgID{dest, dest.Protocol + " " + mID, channel.ID})
 	}
 	return "", nil
 }
@@ -505,7 +539,7 @@ func getProtocol(msg *config.Message) string {
 	return p[0]
 }
 
-func modifyMessageTengo(filename string, msg *config.Message) error {
+func modifyInMessageTengo(filename string, msg *config.Message) error {
 	if filename == "" {
 		return nil
 	}
@@ -517,6 +551,7 @@ func modifyMessageTengo(filename string, msg *config.Message) error {
 	s.SetImports(stdlib.GetModuleMap(stdlib.AllModuleNames()...))
 	_ = s.Add("msgText", msg.Text)
 	_ = s.Add("msgUsername", msg.Username)
+	_ = s.Add("msgUserID", msg.UserID)
 	_ = s.Add("msgAccount", msg.Account)
 	_ = s.Add("msgChannel", msg.Channel)
 	c, err := s.Compile()
@@ -545,6 +580,7 @@ func (gw *Gateway) modifyUsernameTengo(msg *config.Message, br *bridge.Bridge) (
 	_ = s.Add("result", "")
 	_ = s.Add("msgText", msg.Text)
 	_ = s.Add("msgUsername", msg.Username)
+	_ = s.Add("msgUserID", msg.UserID)
 	_ = s.Add("nick", msg.Username)
 	_ = s.Add("msgAccount", msg.Account)
 	_ = s.Add("msgChannel", msg.Channel)
@@ -564,22 +600,28 @@ func (gw *Gateway) modifyUsernameTengo(msg *config.Message, br *bridge.Bridge) (
 	return c.Get("result").String(), nil
 }
 
-func (gw *Gateway) modifySendMessageTengo(origmsg *config.Message, msg *config.Message, br *bridge.Bridge) error {
+func (gw *Gateway) modifyOutMessageTengo(origmsg *config.Message, msg *config.Message, br *bridge.Bridge) (bool, error) {
 	filename := gw.BridgeValues().Tengo.OutMessage
-	var res []byte
-	var err error
+	var (
+		res  []byte
+		err  error
+		drop bool
+	)
+
 	if filename == "" {
 		res, err = internal.Asset("tengo/outmessage.tengo")
 		if err != nil {
-			return err
+			return drop, err
 		}
 	} else {
 		res, err = ioutil.ReadFile(filename)
 		if err != nil {
-			return err
+			return drop, err
 		}
 	}
+
 	s := tengo.NewScript(res)
+
 	s.SetImports(stdlib.GetModuleMap(stdlib.AllModuleNames()...))
 	_ = s.Add("inAccount", origmsg.Account)
 	_ = s.Add("inProtocol", origmsg.Protocol)
@@ -593,14 +635,20 @@ func (gw *Gateway) modifySendMessageTengo(origmsg *config.Message, msg *config.M
 	_ = s.Add("outEvent", msg.Event)
 	_ = s.Add("msgText", msg.Text)
 	_ = s.Add("msgUsername", msg.Username)
+	_ = s.Add("msgUserID", msg.UserID)
+	_ = s.Add("msgDrop", drop)
 	c, err := s.Compile()
 	if err != nil {
-		return err
+		return drop, err
 	}
+
 	if err := c.Run(); err != nil {
-		return err
+		return drop, err
 	}
+
+	drop = c.Get("msgDrop").Bool()
 	msg.Text = c.Get("msgText").String()
 	msg.Username = c.Get("msgUsername").String()
-	return nil
+
+	return drop, nil
 }

gateway/gateway_test.go

@@ -533,7 +533,7 @@ func (s *ignoreTestSuite) TestIgnoreNicks() {
 func BenchmarkTengo(b *testing.B) {
 	msg := &config.Message{Username: "user", Text: "blah testing", Account: "protocol.account", Channel: "mychannel"}
 	for n := 0; n < b.N; n++ {
-		err := modifyMessageTengo("bench.tengo", msg)
+		err := modifyInMessageTengo("bench.tengo", msg)
 		if err != nil {
 			return
 		}

gateway/handlers.go

@@ -169,7 +169,7 @@ func (gw *Gateway) ignoreEvent(event string, dest *bridge.Bridge) bool {
 	switch event {
 	case config.EventAvatarDownload:
 		// Avatar downloads are only relevant for telegram and mattermost for now
-		if dest.Protocol != "mattermost" && dest.Protocol != "telegram" {
+		if dest.Protocol != "mattermost" && dest.Protocol != "telegram" && dest.Protocol != "xmpp" {
 			return true
 		}
 	case config.EventJoinLeave:
@@ -179,7 +179,7 @@ func (gw *Gateway) ignoreEvent(event string, dest *bridge.Bridge) bool {
 		}
 	case config.EventTopicChange:
 		// only relay topic change when used in some way on other side
-		if dest.GetBool("ShowTopicChange") && dest.GetBool("SyncTopic") {
+		if !dest.GetBool("ShowTopicChange") && !dest.GetBool("SyncTopic") {
 			return true
 		}
 	}

gateway/router.go

@@ -160,7 +160,7 @@ func (r *Router) handleReceive() {
 				// For some bridges we always add/update the message ID.
 				// This is necessary as msgIDs will change if a bridge returns
 				// a different ID in response to edits.
-				if !exists || msg.Protocol == "discord" {
+				if !exists {
 					gw.Messages.Add(msg.Protocol+" "+msg.ID, msgIDs)
 				}
 			}

go.mod

@@ -3,64 +3,60 @@ module github.com/42wim/matterbridge
 require (
 	github.com/42wim/go-gitter v0.0.0-20170828205020-017310c2d557
 	github.com/Baozisoftware/qrcode-terminal-go v0.0.0-20170407111555-c0650d8dff0f
-	github.com/Jeffail/gabs v1.1.1 // indirect
-	github.com/Philipp15b/go-steam v1.0.1-0.20190816133340-b04c5a83c1c0
-	github.com/Rhymen/go-whatsapp v0.1.0
-	github.com/d5/tengo/v2 v2.0.2
-	github.com/dfordsoft/golib v0.0.0-20180902042739-76ee6ab99bec
-	github.com/fsnotify/fsnotify v1.4.7
-	github.com/go-telegram-bot-api/telegram-bot-api v4.6.5-0.20181225215658-ec221ba9ea45+incompatible
-	github.com/gomarkdown/markdown v0.0.0-20200127000047-1813ea067497
-	github.com/google/gops v0.3.6
+	github.com/Jeffail/gabs v1.4.0 // indirect
+	github.com/Philipp15b/go-steam v1.0.1-0.20200727090957-6ae9b3c0a560
+	github.com/Rhymen/go-whatsapp v0.1.2-0.20210407153411-c58e164e05b8
+	github.com/SevereCloud/vksdk/v2 v2.9.2
+	github.com/d5/tengo/v2 v2.7.0
+	github.com/davecgh/go-spew v1.1.1
+	github.com/fsnotify/fsnotify v1.4.9
+	github.com/go-telegram-bot-api/telegram-bot-api v1.0.1-0.20200524105306-7434b0456e81
+	github.com/gomarkdown/markdown v0.0.0-20210514010506-3b9f47219fe7
+	github.com/google/gops v0.3.18
 	github.com/gopackage/ddp v0.0.0-20170117053602-652027933df4 // indirect
-	github.com/gorilla/schema v1.1.0
-	github.com/gorilla/websocket v1.4.1
-	github.com/hashicorp/golang-lru v0.5.3
-	github.com/hpcloud/tail v1.0.0 // indirect
+	github.com/gorilla/schema v1.2.0
+	github.com/gorilla/websocket v1.4.2
+	github.com/hashicorp/golang-lru v0.5.4
 	github.com/jpillora/backoff v1.0.0
-	github.com/keybase/go-keybase-chat-bot v0.0.0-20200226211841-4e48f3eaef3e
-	github.com/labstack/echo/v4 v4.1.13
+	github.com/keybase/go-keybase-chat-bot v0.0.0-20200505163032-5cacf52379da
+	github.com/kyokomi/emoji/v2 v2.2.8
+	github.com/labstack/echo/v4 v4.3.0
 	github.com/lrstanley/girc v0.0.0-20190801035559-4fc93959e1a7
-	github.com/matterbridge/Rocket.Chat.Go.SDK v0.0.0-20190210153444-cc9d05784d5d
-	github.com/matterbridge/discordgo v0.18.1-0.20200308151012-aa40f01cbcc3
-	github.com/matterbridge/emoji v2.1.1-0.20191117213217-af507f6b02db+incompatible
-	github.com/matterbridge/go-xmpp v0.0.0-20180529212104-cd19799fba91
-	github.com/matterbridge/gomatrix v0.0.0-20200209224845-c2104d7936a6
-	github.com/matterbridge/gozulipbot v0.0.0-20190212232658-7aa251978a18
-	github.com/matterbridge/logrus-prefixed-formatter v0.0.0-20180806162718-01618749af61
-	github.com/matterbridge/msgraph.go v0.0.0-20200308150230-9e043fe9dbaa
-	github.com/mattermost/mattermost-server v5.5.0+incompatible
-	github.com/mattn/go-runewidth v0.0.7 // indirect
-	github.com/mattn/godown v0.0.0-20180312012330-2e9e17e0ea51
-	github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect
-	github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474 // indirect
+	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
+	github.com/matterbridge/Rocket.Chat.Go.SDK v0.0.0-20210403163225-761e8622445d
+	github.com/matterbridge/discordgo v0.21.2-0.20210201201054-fb39a175b4f7
+	github.com/matterbridge/go-xmpp v0.0.0-20200418225040-c8a3a57b4050
+	github.com/matterbridge/gozulipbot v0.0.0-20200820220548-be5824faa913
+	github.com/matterbridge/logrus-prefixed-formatter v0.5.3-0.20200523233437-d971309a77ba
+	github.com/mattermost/mattermost-server/v5 v5.30.1
+	github.com/mattn/godown v0.0.1
+	github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
+	github.com/missdeer/golib v1.0.4
+	github.com/mreiferson/go-httpclient v0.0.0-20201222173833-5e475fde3a4d // indirect
 	github.com/mrexodia/wray v0.0.0-20160318003008-78a2c1f284ff // indirect
 	github.com/nelsonken/gomf v0.0.0-20180504123937-a9dd2f9deae9
-	github.com/nicksnyder/go-i18n v1.4.0 // indirect
-	github.com/onsi/ginkgo v1.6.0 // indirect
-	github.com/onsi/gomega v1.4.1 // indirect
 	github.com/paulrosania/go-charset v0.0.0-20190326053356-55c9d7a5834c
-	github.com/pborman/uuid v0.0.0-20160216163710-c55201b03606 // indirect
-	github.com/rs/xid v1.2.1
-	github.com/russross/blackfriday v1.5.2
+	github.com/rs/xid v1.3.0
+	github.com/russross/blackfriday v1.6.0
 	github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca
-	github.com/shazow/ssh-chat v1.8.3-0.20200308224626-80ddf1f43a98
-	github.com/sirupsen/logrus v1.4.2
-	github.com/slack-go/slack v0.6.3-0.20200228121756-f56d616d5901
-	github.com/spf13/viper v1.6.1
-	github.com/stretchr/testify v1.4.0
-	github.com/technoweenie/multipartstreamer v1.0.1 // indirect
+	github.com/shazow/ssh-chat v1.10.1
+	github.com/sirupsen/logrus v1.8.1
+	github.com/skip2/go-qrcode v0.0.0-20190110000554-dc11ecdae0a9 // indirect
+	github.com/slack-go/slack v0.9.1
+	github.com/spf13/afero v1.3.4 // indirect
+	github.com/spf13/cast v1.3.1 // indirect
+	github.com/spf13/viper v1.7.1
+	github.com/stretchr/testify v1.7.0
+	github.com/vincent-petithory/dataurl v0.0.0-20191104211930-d1553a71de50
+	github.com/writeas/go-strip-markdown v2.0.1+incompatible
 	github.com/x-cray/logrus-prefixed-formatter v0.5.2 // indirect
-	github.com/zfjagann/golang-ring v0.0.0-20190106091943-a88bb6aef447
-	golang.org/x/image v0.0.0-20191214001246-9130b4cfad52
-	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
-	gopkg.in/fsnotify.v1 v1.4.7 // indirect
-	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
-	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
+	github.com/yaegashi/msgraph.go v0.1.4
+	github.com/zfjagann/golang-ring v0.0.0-20210116075443-7c86fdb43134
+	golang.org/x/image v0.0.0-20210504121937-7319ad40d33e
+	golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c
+	gomod.garykim.dev/nc-talk v0.2.2
+	gopkg.in/olahol/melody.v1 v1.0.0-20170518105555-d52139073376
+	layeh.com/gumble v0.0.0-20200818122324-146f9205029b
 )
 
-//replace github.com/bwmarrin/discordgo v0.20.2 => github.com/matterbridge/discordgo v0.18.1-0.20200109173909-ed873362fa43
-
-//replace github.com/yaegashi/msgraph.go => github.com/matterbridge/msgraph.go v0.0.0-20191226214848-9e5d9c08a4e1
-
-go 1.13
+go 1.15

matterbridge.go

@@ -4,6 +4,7 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"runtime"
 	"strings"
 
 	"github.com/42wim/matterbridge/bridge/config"
@@ -15,7 +16,7 @@ import (
 )
 
 var (
-	version = "1.17.0"
+	version = "1.22.2"
 	githash string
 
 	flagConfig  = flag.String("conf", "matterbridge.toml", "config file")
@@ -50,6 +51,15 @@ func main() {
 	cfg := config.NewConfig(rootLogger, *flagConfig)
 	cfg.BridgeValues().General.Debug = *flagDebug
 
+	// if logging to a file, ensure it is closed when the program terminates
+	// nolint:errcheck
+	defer func() {
+		if f, ok := rootLogger.Out.(*os.File); ok {
+			f.Sync()
+			f.Close()
+		}
+	}()
+
 	r, err := gateway.NewRouter(rootLogger, cfg, bridgemap.FullMap)
 	if err != nil {
 		logger.Fatalf("Starting gateway failed: %s", err)
@@ -67,17 +77,31 @@ func setupLogger() *logrus.Logger {
 		Formatter: &prefixed.TextFormatter{
 			PrefixPadding: 13,
 			DisableColors: true,
-			FullTimestamp: true,
 		},
 		Level: logrus.InfoLevel,
 	}
 	if *flagDebug || os.Getenv("DEBUG") == "1" {
+		logger.SetReportCaller(true)
 		logger.Formatter = &prefixed.TextFormatter{
-			PrefixPadding:   13,
-			DisableColors:   true,
-			FullTimestamp:   false,
-			ForceFormatting: true,
+			PrefixPadding: 13,
+			DisableColors: true,
+			FullTimestamp: false,
+
+			CallerFormatter: func(function, file string) string {
+				return fmt.Sprintf(" [%s:%s]", function, file)
+			},
+			CallerPrettyfier: func(f *runtime.Frame) (string, string) {
+				sp := strings.SplitAfter(f.File, "/matterbridge/")
+				filename := f.File
+				if len(sp) > 1 {
+					filename = sp[1]
+				}
+				s := strings.Split(f.Function, ".")
+				funcName := s[len(s)-1]
+				return funcName, fmt.Sprintf("%s:%d", filename, f.Line)
+			},
 		}
+
 		logger.Level = logrus.DebugLevel
 		logger.WithFields(logrus.Fields{"prefix": "main"}).Info("Enabling debug logging.")
 	}

matterbridge.toml.sample

@@ -9,12 +9,12 @@
 [irc]
 
 #You can configure multiple servers "[irc.name]" or "[irc.name2]"
-#In this example we use [irc.freenode]
+#In this example we use [irc.libera]
 #REQUIRED
-[irc.freenode]
+[irc.libera]
 #irc server to connect to.
 #REQUIRED
-Server="irc.freenode.net:6667"
+Server="irc.libera.chat:6667"
 
 #Password for irc server (if necessary)
 #OPTIONAL (default "")
@@ -24,7 +24,7 @@ Password=""
 #OPTIONAL (default false)
 UseTLS=false
 
-#Enable SASL (PLAIN) authentication. (freenode requires this from eg AWS hosts)
+#Enable SASL (PLAIN) authentication. (libera requires this from eg AWS hosts)
 #It uses NickServNick and NickServPassword as login and password
 #OPTIONAL (default false)
 UseSASL=false
@@ -55,7 +55,7 @@ Charset=""
 #REQUIRED
 Nick="matterbot"
 
-#If you registered your bot with a service like Nickserv on freenode.
+#If you registered your bot with a service like Nickserv on libera.
 #Also being used when UseSASL=true
 #
 #Note: if you want do to quakenet auth, set NickServNick="Q@CServe.quakenet.org"
@@ -76,20 +76,24 @@ MessageDelay=1300
 
 #Maximum amount of messages to hold in queue. If queue is full
 #messages will be dropped.
-#<message clipped> will be add to the message that fills the queue.
+#<clipped message> will be add to the message that fills the queue.
 #OPTIONAL (default 30)
 MessageQueue=30
 
 #Maximum length of message sent to irc server. If it exceeds
-#<message clipped> will be add to the message.
+#<clipped message> will be add to the message.
 #OPTIONAL (default 400)
 MessageLength=400
 
-#Split messages on MessageLength instead of showing the <message clipped>
+#Split messages on MessageLength instead of showing the <clipped message>
 #WARNING: this could lead to flooding
 #OPTIONAL (default false)
 MessageSplit=false
 
+#Message to show when a message is too big
+#Default "<clipped message>"
+MessageClipped="<clipped message>"
+
 #Delay in seconds to rejoin a channel when kicked
 #OPTIONAL (default 0)
 RejoinDelay=0
@@ -103,6 +107,16 @@ ColorNicks=false
 #OPTIONAL (default empty)
 RunCommands=["PRIVMSG user hello","PRIVMSG chanserv something"]
 
+#PingDelay specifies how long to wait to send a ping to the irc server.
+#You can use s for second, m for minute
+#String
+#OPTIONAL (default 1m)
+PingDelay="1m"
+
+#StripMarkdown strips markdown from messages
+#OPTIONAL (default false)
+StripMarkdown=false
+
 #Nicks you want to ignore.
 #Regular expressions supported
 #Messages from those users will not be sent to other bridges.
@@ -177,6 +191,25 @@ StripNick=false
 #OPTIONAL (default false)
 ShowTopicChange=false
 
+#Delay in milliseconds between channel joins
+#Only useful when you have a LOT of channels to join
+#See https://github.com/42wim/matterbridge/issues/1084
+#OPTIONAL (default 0)
+JoinDelay=0
+
+#Use the optional RELAYMSG extension for username spoofing on IRC.
+#This requires an IRCd that supports the draft/relaymsg specification: currently this includes
+#Oragono 2.4.0+ and InspIRCd 3 with the m_relaymsg contrib module.
+#See https://github.com/42wim/matterbridge/issues/667#issuecomment-634214165 for more details.
+#Spoofed nicks will use the configured RemoteNickFormat, replacing reserved IRC characters
+#(!+%@&#$:'"?*,.) with a hyphen (-).
+#On most configurations, the RemoteNickFormat must include a separator character such as "/".
+#You should make sure that the settings here match your IRCd.
+#This option overrides ColorNicks.
+#OPTIONAL (default false)
+UseRelayMsg=false
+#RemoteNickFormat="{NICK}/{PROTOCOL}"
+
 ###################################################################
 #XMPP section
 ###################################################################
@@ -211,6 +244,10 @@ Nick="xmppbot"
 #OPTIONAL (default false)
 SkipTLSVerify=true
 
+#Enable to use plaintext connection to your XMPP server.
+#OPTIONAL (default false)
+NoTLS=true
+
 ## RELOADABLE SETTINGS
 ## Settings below can be reloaded by editing the file
 
@@ -277,6 +314,11 @@ StripNick=false
 #OPTIONAL (default false)
 ShowTopicChange=false
 
+#Enable sending messages using a webhook instead of regular MUC messages.
+#Only works with a prosody server using mod_slack_webhook. Does not support editing.
+#OPTIONAL (default "")
+WebhookURL="https://yourdomain/prosody/msg/someid"
+
 ###################################################################
 #mattermost section
 ###################################################################
@@ -545,6 +587,96 @@ Label=""
 # REQUIRED
 Team="myteam"
 
+###################################################################
+# Microsoft teams section
+# See https://github.com/42wim/matterbridge/wiki/MS-Teams-setup
+###################################################################
+
+[msteams.myteam]
+
+# TenantID
+# See https://github.com/42wim/matterbridge/wiki/MS-Teams-setup#get-necessary-ids-for-matterbridge
+TenantID="xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
+
+# ClientID
+# See https://github.com/42wim/matterbridge/wiki/MS-Teams-setup#get-necessary-ids-for-matterbridge
+ClientID="xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
+
+# TeamID
+# See https://github.com/42wim/matterbridge/wiki/MS-Teams-setup#get-necessary-ids-for-matterbridge
+TeamID="xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
+
+## RELOADABLE SETTINGS
+## Settings below can be reloaded by editing the file
+
+#Nicks you want to ignore.
+#Regular expressions supported
+#Messages from those users will not be sent to other bridges.
+#OPTIONAL
+IgnoreNicks="ircspammer1 ircspammer2"
+
+#Messages you want to ignore.
+#Messages matching these regexp will be ignored and not sent to other bridges
+#See https://regex-golang.appspot.com/assets/html/index.html for more regex info
+#OPTIONAL (example below ignores messages starting with ~~ or messages containing badword
+IgnoreMessages="^~~ badword"
+
+#messages you want to replace.
+#it replaces outgoing messages from the bridge.
+#so you need to place it by the sending bridge definition.
+#regular expressions supported
+#some examples:
+#this replaces cat => dog and sleep => awake
+#replacemessages=[ ["cat","dog"], ["sleep","awake"] ]
+#this replaces every number with number.  123 => numbernumbernumber
+#replacemessages=[ ["[0-9]","number"] ]
+#optional (default empty)
+ReplaceMessages=[ ["cat","dog"] ]
+
+#nicks you want to replace.
+#see replacemessages for syntaxa
+#optional (default empty)
+ReplaceNicks=[ ["user--","user"] ]
+
+#Extractnicks is used to for example rewrite messages from other relaybots
+#See https://github.com/42wim/matterbridge/issues/713 and https://github.com/42wim/matterbridge/issues/466
+#some examples:
+#this replaces a message like "Relaybot: <relayeduser> something interesting" to "relayeduser: something interesting"
+#ExtractNicks=[ [ "Relaybot", "<(.*?)>\\s+" ] ]
+#you can use multiple entries for multiplebots
+#this also replaces a message like "otherbot: (relayeduser) something else" to "relayeduser: something else"
+#ExtractNicks=[ [ "Relaybot", "<(.*?)>\\s+" ],[ "otherbot","\\((.*?)\\)\\s+" ]
+#OPTIONAL (default empty)
+ExtractNicks=[ ["otherbot","<(.*?)>\\s+" ] ]
+
+#extra label that can be used in the RemoteNickFormat
+#optional (default empty)
+Label=""
+
+#RemoteNickFormat defines how remote users appear on this bridge
+#See [general] config section for default options
+RemoteNickFormat="[{PROTOCOL}] <{NICK}> "
+
+#Enable to show users joins/parts from other bridges
+#Currently works for messages from the following bridges: irc, mattermost, slack, discord
+#OPTIONAL (default false)
+ShowJoinPart=false
+
+#StripNick only allows alphanumerical nicks. See https://github.com/42wim/matterbridge/issues/285
+#It will strip other characters from the nick
+#OPTIONAL (default false)
+StripNick=false
+
+#Enable to show topic changes from other bridges
+#Only works hiding/show topic changes from slack bridge for now
+#OPTIONAL (default false)
+ShowTopicChange=false
+
+#Opportunistically preserve threaded replies between bridges
+#that support threading
+#OPTIONAL (default false)
+PreserveThreading=false
+
 ###################################################################
 #slack section
 ###################################################################
@@ -698,6 +830,10 @@ PreserveThreading=false
 #OPTIONAL (default false)
 ShowUserTyping=false
 
+#Message to show when a message is too big
+#Default "<clipped message>"
+MessageClipped="<clipped message>"
+
 ###################################################################
 #discord section
 ###################################################################
@@ -720,6 +856,14 @@ Server="yourservername"
 ## All settings below can be reloaded by editing the file.
 ## They are also all optional.
 
+# AllowMention controls which mentions are allowed. If not specified, all mentions are allowed.
+# Note that even when a mention is not allowed, it will still be displayed nicely and be clickable. It just prevents the ping/notification.
+#
+# "everyone" allows @everyone and @here mentions
+# "roles" allows @role mentions
+# "users" allows @user mentions
+AllowMention=["everyone", "roles", "users"]
+
 # ShowEmbeds shows the title, description and URL of embedded messages (sent by other bots)
 ShowEmbeds=false
 
@@ -736,10 +880,11 @@ UseUserName=false
 # UseDiscriminator appends the `#xxxx` discriminator when used with UseUserName
 UseDiscriminator=false
 
-# WebhookURL sends messages in the style of puppets.
-# This only works if you have one discord channel, if you have multiple discord channels you'll have to specify it in the gateway config
-# Example: "https://discordapp.com/api/webhooks/1234/abcd_xyzw"
-WebhookURL=""
+# AutoWebhooks automatically configures message sending in the style of puppets.
+# This is an easier alternative to manually configuring "WebhookURL" for each gateway,
+# as turning this on will automatically load or create webhooks for each channel.
+# This feature requires the "Manage Webhooks" permission (either globally or as per-channel).
+AutoWebhooks=false
 
 # EditDisable disables sending of edits to other bridges
 EditDisable=false
@@ -824,6 +969,10 @@ ShowTopicChange=false
 # Supported from the following bridges: slack
 SyncTopic=false
 
+#Message to show when a message is too big
+#Default "<clipped message>"
+MessageClipped="<clipped message>"
+
 ###################################################################
 #telegram section
 ###################################################################
@@ -882,6 +1031,13 @@ QuoteFormat="{MESSAGE} (re @{QUOTENICK}: {QUOTEMESSAGE})"
 #OPTIONAL (default false)
 MediaConvertWebPToPNG=false
 
+#Convert Tgs (Telegram animated sticker) images to PNG before upload.
+#This is useful when your bridge also contains platforms that do not support animated WebP files, like Discord.
+#This requires the external dependency `lottie`, which can be installed like this:
+#`pip install lottie cairosvg`
+#https://github.com/42wim/matterbridge/issues/874
+#MediaConvertTgs="png"
+
 #Disable sending of edits to other bridges
 #OPTIONAL (default false)
 EditDisable=false
@@ -1101,21 +1257,33 @@ ShowTopicChange=false
 #REQUIRED
 Server="https://matrix.org"
 
-#login/pass of your bot.
+#Authentication for your bot.
+#You can use either login/password OR mxid/token. The latter will be preferred if found.
 #Use a dedicated user for this and not your own!
 #Messages sent from this user will not be relayed to avoid loops.
 #REQUIRED
 Login="yourlogin"
 Password="yourpass"
+#OR
+MxID="@yourlogin:domain.tld"
+Token="tokenforthebotuser"
 
 #Whether to send the homeserver suffix. eg ":matrix.org" in @username:matrix.org
 #to other bridges, or only send "username".(true only sends username)
 #OPTIONAL (default false)
 NoHomeServerSuffix=false
 
+#Whether to disable sending of HTML content to matrix
+#See https://github.com/42wim/matterbridge/issues/1022
+#OPTIONAL (default false)
+HTMLDisable=false
+
 ## RELOADABLE SETTINGS
 ## Settings below can be reloaded by editing the file
 
+# UseUserName shows the username instead of the server nickname
+UseUserName=false
+
 #Whether to prefix messages from other bridges to matrix with the sender's nick.
 #Useful if username overrides for incoming webhooks isn't enabled on the
 #matrix server. If you set PrefixMessagesWithNick to true, each message
@@ -1278,12 +1446,92 @@ StripNick=false
 #OPTIONAL (default false)
 ShowTopicChange=false
 
+###################################################################
+# NCTalk (Nextcloud Talk)
+###################################################################
 
+[nctalk.bridge]
+
+# Url of your Nextcloud server
+Server = "https://cloud.youdomain.me"
+
+# Enable to not verify the certificate on your Nextcloud server.
+# e.g. when using selfsigned certificates
+# OPTIONAL (default false)
+SkipTLSVerify=true
+
+# Username of the bot
+Login = "talkuser"
+
+# Password of the bot
+Password = "talkuserpass"
+
+# Suffix for Guest Users
+GuestSuffix = " (Guest)"
 
 ###################################################################
+# Mumble
+###################################################################
+
+[mumble.bridge]
+
+# Host and port of your Mumble server
+Server = "mumble.yourdomain.me:64738"
+
+# Nickname to log in as
+Nick = "matterbridge"
+
+# Some servers require a password
+# OPTIONAL (default empty)
+Password = "serverpasswordhere"
+
+# User comment to set on the Mumble user, visible to other users.
+# OPTIONAL (default empty)
+UserComment="I am bridging text messages between this channel and #general on irc.yourdomain.me"
+
+# Self-signed TLS client certificate + private key used to connect to
+# Mumble.  This is required if you want to register the matterbridge
+# user on your Mumble server, so its nick becomes reserved.
+# You can generate a keypair using e.g.
 #
+#     openssl req -x509 -newkey rsa:2048 -nodes -days 10000 \
+#             -keyout mumble.key -out mumble.crt
+#
+# To actually register the matterbridege user, connect to Mumble as an
+# admin, right click on the user and click "Register".
+#
+# OPTIONAL (default empty)
+TLSClientCertificate="mumble.crt"
+TLSClientKey="mumble.key"
+
+# TLS CA certificate used to validate the Mumble server.
+# OPTIONAL (defaults to Go system CA)
+TLSCACertificate=mumble-ca.crt
+
+# Enable to not verify the certificate on your Mumble server.
+# e.g. when using selfsigned certificates
+# OPTIONAL (default false)
+SkipTLSVerify=false
+
+#Message to show when a message is too big
+#Default "<clipped message>"
+MessageClipped="<clipped message>"
+
+###################################################################
+#VK
+###################################################################
+#
+[vk.myvk]
+#Group access token
+#See https://vk.com/dev/bots_docs
+Token="Yourtokenhere"
+
+#Group ID
+#For example in URL https://vk.com/public168963511 group ID is 168963511
+GroupID=123456789
+
+###################################################################
 # WhatsApp
-#
 ###################################################################
 
 [whatsapp.bridge]
@@ -1310,9 +1558,7 @@ Label="Organization"
 
 
 ###################################################################
-#
 # zulip
-#
 ###################################################################
 
 [zulip]
@@ -1420,6 +1666,8 @@ Buffer=1000
 
 #Bearer token used for authentication
 #curl -H "Authorization: Bearer token" http://localhost:4242/api/messages
+# https://github.com/vi/websocat
+# websocat -H="Authorization: Bearer token" ws://127.0.0.1:4242/api/websocket
 #OPTIONAL (no authorization if token is empty)
 Token="mytoken"
 
@@ -1443,7 +1691,8 @@ RemoteNickFormat="{NICK}"
 ## Settings below can be reloaded by editing the file
 
 #RemoteNickFormat defines how remote users appear on this bridge
-#The string "{NICK}" (case sensitive) will be replaced by the actual nick / username.
+#The string "{NICK}" (case sensitive) will be replaced by the actual nick.
+#The string "{USERID}" (case sensitive) will be replaced by the user ID.
 #The string "{BRIDGE}" (case sensitive) will be replaced by the sending bridge
 #The string "{LABEL}" (case sensitive) will be replaced by label= field of the sending bridge
 #The string "{PROTOCOL}" (case sensitive) will be replaced by the protocol used by the bridge
@@ -1499,6 +1748,14 @@ MediaDownloadBlacklist=[".html$",".htm$"]
 #OPTIONAL (default false)
 IgnoreFailureOnStart=false
 
+#LogFile defines the location of a file to write logs into, rather
+#than stdout.
+#Logging will still happen on stdout if the file cannot be open for
+#writing, or if the value is empty. Note that the log won't roll, so
+#you might want to use logrotate(8) with this feature.
+#OPTIONAL (default empty)
+LogFile="/var/log/matterbridge.log"
+
 ###################################################################
 #Tengo configuration
 ###################################################################
@@ -1510,7 +1767,7 @@ IgnoreFailureOnStart=false
 #This script will receive every incoming message and can be used to modify the Username and the Text of that message.
 #The script will have the following global variables:
 #to modify: msgUsername and msgText
-#to read: msgChannel and msgAccount
+#to read: msgUserID, msgChannel, msgAccount
 #
 #The script is reloaded on every message, so you can modify the script on the fly.
 #
@@ -1534,9 +1791,12 @@ InMessage="example.tengo"
 #read-only:
 #inAccount, inProtocol, inChannel, inGateway, inEvent
 #outAccount, outProtocol, outChannel, outGateway, outEvent
+#msgUserID
 #
 #read-write:
-#msgText, msgUsername
+#msgText, msgUsername, msgDrop
+#
+#msgDrop is a bool which is default false, when set true this message will be dropped
 #
 #The script is reloaded on every message, so you can modify the script on the fly.
 #
@@ -1549,7 +1809,7 @@ OutMessage="example.tengo"
 #RemoteNickFormat allows you to specify the location of a tengo (https://github.com/d5/tengo/) script.
 #The script will have the following global variables:
 #to modify: result
-#to read: channel, bridge, gateway, protocol, nick
+#to read: channel, bridge, gateway, protocol, nick, msgUserID
 #
 #The result will be set in {TENGO} in the RemoteNickFormat key of every bridge where {TENGO} is specified
 #
@@ -1587,34 +1847,52 @@ enable=true
 
     # account specified above
     # REQUIRED
-    account="irc.freenode"
+    account="irc.libera"
 
-    # channel to connect on that account
-    # How to specify them for the different bridges:
+    # The channel key in each gateway is mapped to a similar group chat ID on the chat platform
+    # To find the group chat ID for different platforms, refer to the table below
     #
-    # irc        - #channel (# is required) (this needs to be lowercase!)
-    # mattermost - channel (the channel name as seen in the URL, not the displayname)
-    # gitter     - username/room
-    # xmpp       - channel
-    # slack      - channel (without the #)
-    #            - ID:C123456 (where C123456 is the channel ID) does not work with webhook
-    # discord    - channel (without the #)
-    #            - ID:123456789 (where 123456789 is the channel ID)
-    #               (https://github.com/42wim/matterbridge/issues/57)
-    #            - category/channel (without the #) if you're using discord categories to group your channels
-    # telegram   - chatid (a large negative number, eg -123456789)
-    #             see (https://www.linkedin.com/pulse/telegram-bots-beginners-marco-frau)
-    # hipchat    - id_channel (see https://www.hipchat.com/account/xmpp for the correct channel)
-    # rocketchat - #channel (# is required (also needed for private channels!)
-    # matrix     - #channel:server (eg #yourchannel:matrix.org)
-    #            - encrypted rooms are not supported in matrix
-    # steam      - chatid (a large number).
-    #             The number in the URL when you click "enter chat room" in the browser
-    # whatsapp   - 48111222333-123455678999@g.us A unique group JID;
-    #              if you specify an empty string bridge will list all the possibilities
-    #            - "Group Name" if you specify a group name the bridge will hint its JID to specify
-    #              as group names might change in time and contain weird emoticons
-    # zulip      - stream/topic:topicname (without the #)
+    # Platform   |   Identifier name  |            Example            | Description
+    # -------------------------------------------------------------------------------------------------------------------------------------
+    #            |      channel       |            general            | Do not include the # symbol
+    #  discord   |    channel id      |          ID:123456789         | See https://github.com/42wim/matterbridge/issues/57
+    #            | category/channel   |          Media/gaming         | Without # symbol. If you're using discord categories to group your channels
+    # -------------------------------------------------------------------------------------------------------------------------------------
+    #   gitter   |  username/room     |            general            | As seen in the gitter.im URL
+    # -------------------------------------------------------------------------------------------------------------------------------------
+    #   hipchat  |    id_channel      |         example needed        | See https://www.hipchat.com/account/xmpp for the correct channel
+    # -------------------------------------------------------------------------------------------------------------------------------------
+    #    irc     |      channel       |            #general           | The # symbol is required and should be lowercase!
+    # -------------------------------------------------------------------------------------------------------------------------------------
+    # mattermost |      channel       |            general            | This is the channel name as seen in the URL, not the display name
+    # -------------------------------------------------------------------------------------------------------------------------------------
+    #   matrix   | #channel:server    |    #yourchannel:matrix.org    | Encrypted rooms are not supported in matrix
+    # -------------------------------------------------------------------------------------------------------------------------------------
+    #   msteams  |      threadId      |    19:82abcxx@thread.skype    | You'll find the threadId in the URL
+    # -------------------------------------------------------------------------------------------------------------------------------------
+    #   mumble   |    channel id      |              42               | The channel ID, as shown in the channel's "Edit" window
+    # -------------------------------------------------------------------------------------------------------------------------------------
+    # rocketchat |      channel       |            #channel           | # is required for private channels too
+    # -------------------------------------------------------------------------------------------------------------------------------------
+    #   slack    |   channel name     |            general            | Do not include the # symbol
+    #            |    channel id      |           ID:C123456          | The underlying ID of a channel. This doesn't work with webhooks.
+    # -------------------------------------------------------------------------------------------------------------------------------------
+    #   steam    |      chatid        |         example needed        | The number in the URL when you click "enter chat room" in the browser
+    # -------------------------------------------------------------------------------------------------------------------------------------
+    #   nctalk   |      token         |           xs25tz5y            | The token in the URL when you are in a chat. It will be the last part of the URL.
+    # -------------------------------------------------------------------------------------------------------------------------------------
+    #  telegram  |      chatid        |          -123456789           | A large negative number. see https://www.linkedin.com/pulse/telegram-bots-beginners-marco-frau
+    # -------------------------------------------------------------------------------------------------------------------------------------
+    #  vk        |      peerid        |          2000000002           | A number that starts form 2000000000. Use --debug and send any message in chat to get PeerID in the logs 
+    # -------------------------------------------------------------------------------------------------------------------------------------
+    #  whatsapp  |     group JID      | 48111222333-123455678999@g.us | A unique group JID. If you specify an empty string, bridge will list all the possibilities
+    #            |    "Group Name"    |         "Family Chat"         | if you specify a group name, the bridge will find hint the JID to specify. Names can change over time and are not stable.
+    # -------------------------------------------------------------------------------------------------------------------------------------
+    #    xmpp    |      channel       |            general            | The room name
+    # -------------------------------------------------------------------------------------------------------------------------------------
+    #   zulip    | stream/topic:topic |     general/off-topic:food    | Do not use the # when specifying a topic
+    # -------------------------------------------------------------------------------------------------------------------------------------
+
     #
     # REQUIRED
     channel="#testing"
@@ -1627,7 +1905,7 @@ enable=true
 
     #[[gateway.out]] specifies the account and channels we will sent messages to.
     [[gateway.out]]
-    account="irc.freenode"
+    account="irc.libera"
     channel="#testing"
 
         #OPTIONAL - only used for IRC and XMPP protocols at the moment
@@ -1646,13 +1924,16 @@ enable=true
         #OPTIONAL - your irc / xmpp channel key
         key="yourkey"
 
+    # Discord specific gateway options
     [[gateway.inout]]
     account="discord.game"
     channel="mygreatgame"
 
-        #OPTIONAL - webhookurl only works for discord (it needs a different URL for each cahnnel)
         [gateway.inout.options]
-        webhookurl="https://discordapp.com/api/webhooks/123456789123456789/C9WPqExYWONPDZabcdef-def1434FGFjstasJX9pYht73y"
+        # WebhookURL sends messages in the style of "puppets". You must configure a webhook URL for each channel you want to bridge.
+        # If you have more than one channel and don't wnat to configure each channel manually, see the "AutoWebhooks" option in the gateway config.
+        # Example: "https://discord.com/api/webhooks/1234/abcd_xyzw"
+        WebhookURL=""
 
     [[gateway.inout]]
     account="zulip.streamchat"

matterclient/channels.go

@@ -4,7 +4,7 @@ import (
 	"errors"
 	"strings"
 
-	"github.com/mattermost/mattermost-server/model"
+	"github.com/mattermost/mattermost-server/v5/model"
 )
 
 // GetChannels returns all channels we're members off
@@ -167,7 +167,7 @@ func (m *MMClient) JoinChannel(channelId string) error { //nolint:golint
 }
 
 func (m *MMClient) UpdateChannelsTeam(teamID string) error {
-	mmchannels, resp := m.Client.GetChannelsForTeamForUser(teamID, m.User.Id, "")
+	mmchannels, resp := m.Client.GetChannelsForTeamForUser(teamID, m.User.Id, false, "")
 	if resp.Error != nil {
 		return errors.New(resp.Error.DetailedError)
 	}

matterclient/helpers.go

@@ -13,7 +13,7 @@ import (
 
 	"github.com/gorilla/websocket"
 	"github.com/jpillora/backoff"
-	"github.com/mattermost/mattermost-server/model"
+	"github.com/mattermost/mattermost-server/v5/model"
 )
 
 func (m *MMClient) doLogin(firstConnection bool, b *backoff.Backoff) error {
@@ -154,7 +154,7 @@ func (m *MMClient) initUser() error {
 
 		t := &Team{Team: team, Users: usermap, Id: team.Id}
 
-		mmchannels, resp := m.Client.GetChannelsForTeamForUser(team.Id, m.User.Id, "")
+		mmchannels, resp := m.Client.GetChannelsForTeamForUser(team.Id, m.User.Id, false, "")
 		if resp.Error != nil {
 			return resp.Error
 		}

matterclient/matterclient.go

@@ -11,7 +11,7 @@ import (
 	lru "github.com/hashicorp/golang-lru"
 	"github.com/jpillora/backoff"
 	prefixed "github.com/matterbridge/logrus-prefixed-formatter"
-	"github.com/mattermost/mattermost-server/model"
+	"github.com/mattermost/mattermost-server/v5/model"
 	"github.com/sirupsen/logrus"
 )
 
@@ -69,6 +69,7 @@ type MMClient struct {
 	logger     *logrus.Entry
 	rootLogger *logrus.Logger
 	lruCache   *lru.Cache
+	allevents  bool
 }
 
 // New will instantiate a new Matterclient with the specified login details without connecting.
@@ -119,6 +120,10 @@ func (m *MMClient) SetLogLevel(level string) {
 	}
 }
 
+func (m *MMClient) EnableAllEvents() {
+	m.allevents = true
+}
+
 // Login tries to connect the client with the loging details with which it was initialized.
 func (m *MMClient) Login() error {
 	// check if this is a first connect or a reconnection
@@ -220,6 +225,10 @@ func (m *MMClient) WsReceiver() {
 					continue
 				}
 			}
+			if m.allevents {
+				m.MessageChan <- msg
+				continue
+			}
 			switch msg.Raw.Event {
 			case model.WEBSOCKET_EVENT_USER_ADDED,
 				model.WEBSOCKET_EVENT_USER_REMOVED,

matterclient/messages.go

@@ -3,7 +3,7 @@ package matterclient
 import (
 	"strings"
 
-	"github.com/mattermost/mattermost-server/model"
+	"github.com/mattermost/mattermost-server/v5/model"
 )
 
 func (m *MMClient) parseActionPost(rmsg *Message) {

matterclient/users.go

@@ -4,7 +4,7 @@ import (
 	"errors"
 	"time"
 
-	"github.com/mattermost/mattermost-server/model"
+	"github.com/mattermost/mattermost-server/v5/model"
 )
 
 func (m *MMClient) GetNickName(userId string) string { //nolint:golint

tgs.Dockerfile

@@ -0,0 +1,35 @@
+FROM alpine AS builder
+
+COPY . /go/src/matterbridge
+RUN apk add \
+    go \
+    git \
+  && cd /go/src/matterbridge \
+  && go build -mod vendor -ldflags "-X main.githash=$(git log --pretty=format:'%h' -n 1)" -o /bin/matterbridge
+
+FROM alpine
+RUN apk --no-cache add \
+    ca-certificates \
+    cairo \
+    libjpeg-turbo \
+    libwebp-dev \
+    mailcap \
+    py3-webencodings \
+    python3 \
+  && apk --no-cache add --virtual .compile \
+    gcc \
+    libffi-dev \
+    libjpeg-turbo-dev \
+    musl-dev \
+    py3-pip \
+    py3-wheel \
+    python3-dev \
+    zlib-dev \
+  && pip3 install --no-cache-dir lottie[PNG] \
+  && apk --no-cache del .compile
+
+COPY --from=builder /bin/matterbridge /bin/matterbridge
+RUN mkdir /etc/matterbridge \
+  && touch /etc/matterbridge/matterbridge.toml \
+  && ln -sf /matterbridge.toml /etc/matterbridge/matterbridge.toml
+ENTRYPOINT ["/bin/matterbridge", "-conf", "/etc/matterbridge/matterbridge.toml"]

vendor/github.com/Jeffail/gabs/README.md

@@ -7,21 +7,17 @@ It does nothing spectacular except for being fabulous.
 
 https://godoc.org/github.com/Jeffail/gabs
 
-## How to install:
+## Install
 
 ``` bash
 go get github.com/Jeffail/gabs
 ```
 
-## How to use
+## Use
 
 ### Parsing and searching JSON
 
 ``` go
-...
-
-import "github.com/Jeffail/gabs"
-
 jsonParsed, err := gabs.ParseJSON([]byte(`{
 	"outter":{
 		"inner":{
@@ -29,7 +25,10 @@ jsonParsed, err := gabs.ParseJSON([]byte(`{
 			"value2":22
 		},
 		"alsoInner":{
-			"value1":20
+			"value1":20,
+			"array1":[
+				30, 40
+			]
 		}
 	}
 }`))
@@ -43,26 +42,26 @@ value, ok = jsonParsed.Path("outter.inner.value1").Data().(float64)
 value, ok = jsonParsed.Search("outter", "inner", "value1").Data().(float64)
 // value == 10.0, ok == true
 
+gObj, err := jsonParsed.JSONPointer("/outter/alsoInner/array1/1")
+if err != nil {
+	panic(err)
+}
+value, ok = gObj.Data().(float64)
+// value == 40.0, ok == true
+
 value, ok = jsonParsed.Path("does.not.exist").Data().(float64)
 // value == 0.0, ok == false
 
 exists := jsonParsed.Exists("outter", "inner", "value1")
 // exists == true
 
-exists := jsonParsed.Exists("does", "not", "exist")
-// exists == false
-
 exists := jsonParsed.ExistsP("does.not.exist")
 // exists == false
-
-...
 ```
 
 ### Iterating objects
 
 ``` go
-...
-
 jsonParsed, _ := gabs.ParseJSON([]byte(`{"object":{ "first": 1, "second": 2, "third": 3 }}`))
 
 // S is shorthand for Search
@@ -70,24 +69,25 @@ children, _ := jsonParsed.S("object").ChildrenMap()
 for key, child := range children {
 	fmt.Printf("key: %v, value: %v\n", key, child.Data().(string))
 }
-
-...
 ```
 
 ### Iterating arrays
 
 ``` go
-...
-
-jsonParsed, _ := gabs.ParseJSON([]byte(`{"array":[ "first", "second", "third" ]}`))
+jsonParsed, err := gabs.ParseJSON([]byte(`{"array":[ "first", "second", "third" ]}`))
+if err != nil {
+	panic(err)
+}
 
 // S is shorthand for Search
-children, _ := jsonParsed.S("array").Children()
+children, err := jsonParsed.S("array").Children()
+if err != nil {
+	panic(err)
+}
+
 for _, child := range children {
 	fmt.Println(child.Data().(string))
 }
-
-...
 ```
 
 Will print:
@@ -108,12 +108,11 @@ objects within the array, this returns a JSON array containing the results for
 each element.
 
 ``` go
-...
-
-jsonParsed, _ := gabs.ParseJSON([]byte(`{"array":[ {"value":1}, {"value":2}, {"value":3} ]}`))
+jsonParsed, err := gabs.ParseJSON([]byte(`{"array":[ {"value":1}, {"value":2}, {"value":3} ]}`))
+if err != nil {
+	panic(err)
+}
 fmt.Println(jsonParsed.Path("array.value").String())
-
-...
 ```
 
 Will print:
@@ -125,8 +124,6 @@ Will print:
 ### Generating JSON
 
 ``` go
-...
-
 jsonObj := gabs.New()
 // or gabs.Consume(jsonObject) to work on an existing map[string]interface{}
 
@@ -135,8 +132,6 @@ jsonObj.SetP(20, "outter.inner.value2")
 jsonObj.Set(30, "outter", "inner2", "value3")
 
 fmt.Println(jsonObj.String())
-
-...
 ```
 
 Will print:
@@ -148,11 +143,7 @@ Will print:
 To pretty-print:
 
 ``` go
-...
-
 fmt.Println(jsonObj.StringIndent("", "  "))
-
-...
 ```
 
 Will print:
@@ -174,8 +165,6 @@ Will print:
 ### Generating Arrays
 
 ``` go
-...
-
 jsonObj := gabs.New()
 
 jsonObj.Array("foo", "array")
@@ -186,8 +175,6 @@ jsonObj.ArrayAppend(20, "foo", "array")
 jsonObj.ArrayAppend(30, "foo", "array")
 
 fmt.Println(jsonObj.String())
-
-...
 ```
 
 Will print:
@@ -199,8 +186,6 @@ Will print:
 Working with arrays by index:
 
 ``` go
-...
-
 jsonObj := gabs.New()
 
 // Create an array with the length of 3
@@ -217,8 +202,6 @@ jsonObj.S("foo").Index(2).SetIndex(2, 1)
 jsonObj.S("foo").Index(2).SetIndex(3, 2)
 
 fmt.Println(jsonObj.String())
-
-...
 ```
 
 Will print:
@@ -232,8 +215,6 @@ Will print:
 This is the easiest part:
 
 ``` go
-...
-
 jsonParsedObj, _ := gabs.ParseJSON([]byte(`{
 	"outter":{
 		"values":{
@@ -246,15 +227,11 @@ jsonParsedObj, _ := gabs.ParseJSON([]byte(`{
 
 jsonOutput := jsonParsedObj.String()
 // Becomes `{"outter":{"values":{"first":10,"second":11}},"outter2":"hello world"}`
-
-...
 ```
 
 And to serialize a specific segment is as simple as:
 
 ``` go
-...
-
 jsonParsedObj := gabs.ParseJSON([]byte(`{
 	"outter":{
 		"values":{
@@ -267,8 +244,6 @@ jsonParsedObj := gabs.ParseJSON([]byte(`{
 
 jsonOutput := jsonParsedObj.Search("outter").String()
 // Becomes `{"values":{"first":10,"second":11}}`
-
-...
 ```
 
 ### Merge two containers

vendor/github.com/Jeffail/gabs/gabs.go

@@ -20,58 +20,85 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 */
 
-// Package gabs implements a simplified wrapper around creating and parsing JSON.
+// Package gabs implements a simplified wrapper around creating and parsing
+// unknown or dynamic JSON.
 package gabs
 
 import (
 	"bytes"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"io"
 	"io/ioutil"
+	"strconv"
 	"strings"
 )
 
-//--------------------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
 
 var (
-	// ErrOutOfBounds - Index out of bounds.
+	// ErrOutOfBounds indicates an index was out of bounds.
 	ErrOutOfBounds = errors.New("out of bounds")
 
-	// ErrNotObjOrArray - The target is not an object or array type.
+	// ErrNotObjOrArray is returned when a target is not an object or array type
+	// but needs to be for the intended operation.
 	ErrNotObjOrArray = errors.New("not an object or array")
 
-	// ErrNotObj - The target is not an object type.
+	// ErrNotObj is returned when a target is not an object but needs to be for
+	// the intended operation.
 	ErrNotObj = errors.New("not an object")
 
-	// ErrNotArray - The target is not an array type.
+	// ErrNotArray is returned when a target is not an array but needs to be for
+	// the intended operation.
 	ErrNotArray = errors.New("not an array")
 
-	// ErrPathCollision - Creating a path failed because an element collided with an existing value.
+	// ErrPathCollision is returned when creating a path failed because an
+	// element collided with an existing value.
 	ErrPathCollision = errors.New("encountered value collision whilst building path")
 
-	// ErrInvalidInputObj - The input value was not a map[string]interface{}.
+	// ErrInvalidInputObj is returned when the input value was not a
+	// map[string]interface{}.
 	ErrInvalidInputObj = errors.New("invalid input object")
 
-	// ErrInvalidInputText - The input data could not be parsed.
+	// ErrInvalidInputText is returned when the input data could not be parsed.
 	ErrInvalidInputText = errors.New("input text could not be parsed")
 
-	// ErrInvalidPath - The filepath was not valid.
+	// ErrInvalidPath is returned when the filepath was not valid.
 	ErrInvalidPath = errors.New("invalid file path")
 
-	// ErrInvalidBuffer - The input buffer contained an invalid JSON string
+	// ErrInvalidBuffer is returned when the input buffer contained an invalid
+	// JSON string.
 	ErrInvalidBuffer = errors.New("input buffer contained invalid JSON")
 )
 
-//--------------------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
 
-// Container - an internal structure that holds a reference to the core interface map of the parsed
-// json. Use this container to move context.
+func resolveJSONPointerHierarchy(path string) ([]string, error) {
+	if len(path) < 1 {
+		return nil, errors.New("failed to resolve JSON pointer: path must not be empty")
+	}
+	if path[0] != '/' {
+		return nil, errors.New("failed to resolve JSON pointer: path must begin with '/'")
+	}
+	hierarchy := strings.Split(path, "/")[1:]
+	for i, v := range hierarchy {
+		v = strings.Replace(v, "~1", "/", -1)
+		v = strings.Replace(v, "~0", "~", -1)
+		hierarchy[i] = v
+	}
+	return hierarchy, nil
+}
+
+//------------------------------------------------------------------------------
+
+// Container references a specific element within a JSON structure.
 type Container struct {
 	object interface{}
 }
 
-// Data - Return the contained data as an interface{}.
+// Data returns the underlying interface{} of the target element in the JSON
+// structure.
 func (g *Container) Data() interface{} {
 	if g == nil {
 		return nil
@@ -79,17 +106,18 @@ func (g *Container) Data() interface{} {
 	return g.object
 }
 
-//--------------------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
 
-// Path - Search for a value using dot notation.
+// Path searches the JSON structure following a path in dot notation.
 func (g *Container) Path(path string) *Container {
 	return g.Search(strings.Split(path, ".")...)
 }
 
-// Search - Attempt to find and return an object within the JSON structure by specifying the
-// hierarchy of field names to locate the target. If the search encounters an array and has not
-// reached the end target then it will iterate each object of the array for the target and return
-// all of the results in a JSON array.
+// Search attempts to find and return an object within the JSON structure by
+// following a provided hierarchy of field names to locate the target. If the
+// search encounters an array and has not reached the end target then it will
+// iterate each object of the array for the target and return all of the results
+// in a JSON array.
 func (g *Container) Search(hierarchy ...string) *Container {
 	var object interface{}
 
@@ -120,22 +148,55 @@ func (g *Container) Search(hierarchy ...string) *Container {
 	return &Container{object}
 }
 
-// S - Shorthand method, does the same thing as Search.
+// JSONPointer parses a JSON pointer path (https://tools.ietf.org/html/rfc6901)
+// and either returns a *gabs.Container containing the result or an error if the
+// referenced item could not be found.
+func (g *Container) JSONPointer(path string) (*Container, error) {
+	hierarchy, err := resolveJSONPointerHierarchy(path)
+	if err != nil {
+		return nil, err
+	}
+
+	object := g.Data()
+	for target := 0; target < len(hierarchy); target++ {
+		pathSeg := hierarchy[target]
+		if mmap, ok := object.(map[string]interface{}); ok {
+			object, ok = mmap[pathSeg]
+			if !ok {
+				return nil, fmt.Errorf("failed to resolve JSON pointer: index '%v' value '%v' was not found", target, pathSeg)
+			}
+		} else if marray, ok := object.([]interface{}); ok {
+			index, err := strconv.Atoi(pathSeg)
+			if err != nil {
+				return nil, fmt.Errorf("failed to resolve JSON pointer: could not parse index '%v' value '%v' into array index: %v", target, pathSeg, err)
+			}
+			if len(marray) <= index {
+				return nil, fmt.Errorf("failed to resolve JSON pointer: index '%v' value '%v' exceeded target array size of '%v'", target, pathSeg, len(marray))
+			}
+			object = marray[index]
+		} else {
+			return &Container{nil}, fmt.Errorf("failed to resolve JSON pointer: index '%v' field '%v' was not found", target, pathSeg)
+		}
+	}
+	return &Container{object}, nil
+}
+
+// S is a shorthand alias for Search.
 func (g *Container) S(hierarchy ...string) *Container {
 	return g.Search(hierarchy...)
 }
 
-// Exists - Checks whether a path exists.
+// Exists checks whether a path exists.
 func (g *Container) Exists(hierarchy ...string) bool {
 	return g.Search(hierarchy...) != nil
 }
 
-// ExistsP - Checks whether a dot notation path exists.
+// ExistsP checks whether a dot notation path exists.
 func (g *Container) ExistsP(path string) bool {
 	return g.Exists(strings.Split(path, ".")...)
 }
 
-// Index - Attempt to find and return an object within a JSON array by index.
+// Index attempts to find and return an element within a JSON array by an index.
 func (g *Container) Index(index int) *Container {
 	if array, ok := g.Data().([]interface{}); ok {
 		if index >= len(array) {
@@ -146,9 +207,9 @@ func (g *Container) Index(index int) *Container {
 	return &Container{nil}
 }
 
-// Children - Return a slice of all the children of the array. This also works for objects, however,
-// the children returned for an object will NOT be in order and you lose the names of the returned
-// objects this way.
+// Children returns a slice of all children of an array element. This also works
+// for objects, however, the children returned for an object will be in a random
+// order and you lose the names of the returned objects this way.
 func (g *Container) Children() ([]*Container, error) {
 	if array, ok := g.Data().([]interface{}); ok {
 		children := make([]*Container, len(array))
@@ -167,7 +228,7 @@ func (g *Container) Children() ([]*Container, error) {
 	return nil, ErrNotObjOrArray
 }
 
-// ChildrenMap - Return a map of all the children of an object.
+// ChildrenMap returns a map of all the children of an object element.
 func (g *Container) ChildrenMap() (map[string]*Container, error) {
 	if mmap, ok := g.Data().(map[string]interface{}); ok {
 		children := map[string]*Container{}
@@ -179,11 +240,11 @@ func (g *Container) ChildrenMap() (map[string]*Container, error) {
 	return nil, ErrNotObj
 }
 
-//--------------------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
 
-// Set - Set the value of a field at a JSON path, any parts of the path that do not exist will be
-// constructed, and if a collision occurs with a non object type whilst iterating the path an error
-// is returned.
+// Set the value of a field at a JSON path, any parts of the path that do not
+// exist will be constructed, and if a collision occurs with a non object type
+// whilst iterating the path an error is returned.
 func (g *Container) Set(value interface{}, path ...string) (*Container, error) {
 	if len(path) == 0 {
 		g.object = value
@@ -209,12 +270,14 @@ func (g *Container) Set(value interface{}, path ...string) (*Container, error) {
 	return &Container{object}, nil
 }
 
-// SetP - Does the same as Set, but using a dot notation JSON path.
+// SetP sets the value of a field at a JSON path using dot notation, any parts
+// of the path that do not exist will be constructed, and if a collision occurs
+// with a non object type whilst iterating the path an error is returned.
 func (g *Container) SetP(value interface{}, path string) (*Container, error) {
 	return g.Set(value, strings.Split(path, ".")...)
 }
 
-// SetIndex - Set a value of an array element based on the index.
+// SetIndex attempts to set a value of an array element based on an index.
 func (g *Container) SetIndex(value interface{}, index int) (*Container, error) {
 	if array, ok := g.Data().([]interface{}); ok {
 		if index >= len(array) {
@@ -226,60 +289,112 @@ func (g *Container) SetIndex(value interface{}, index int) (*Container, error) {
 	return &Container{nil}, ErrNotArray
 }
 
-// Object - Create a new JSON object at a path. Returns an error if the path contains a collision
-// with a non object type.
+// SetJSONPointer parses a JSON pointer path
+// (https://tools.ietf.org/html/rfc6901) and sets the leaf to a value. Returns
+// an error if the pointer could not be resolved due to missing fields.
+func (g *Container) SetJSONPointer(value interface{}, path string) error {
+	hierarchy, err := resolveJSONPointerHierarchy(path)
+	if err != nil {
+		return err
+	}
+
+	if len(hierarchy) == 0 {
+		g.object = value
+		return nil
+	}
+
+	object := g.object
+
+	for target := 0; target < len(hierarchy); target++ {
+		pathSeg := hierarchy[target]
+		if mmap, ok := object.(map[string]interface{}); ok {
+			if target == len(hierarchy)-1 {
+				object = value
+				mmap[pathSeg] = object
+			} else if object = mmap[pathSeg]; object == nil {
+				return fmt.Errorf("failed to resolve JSON pointer: index '%v' value '%v' was not found", target, pathSeg)
+			}
+		} else if marray, ok := object.([]interface{}); ok {
+			index, err := strconv.Atoi(pathSeg)
+			if err != nil {
+				return fmt.Errorf("failed to resolve JSON pointer: could not parse index '%v' value '%v' into array index: %v", target, pathSeg, err)
+			}
+			if len(marray) <= index {
+				return fmt.Errorf("failed to resolve JSON pointer: index '%v' value '%v' exceeded target array size of '%v'", target, pathSeg, len(marray))
+			}
+			if target == len(hierarchy)-1 {
+				object = value
+				marray[index] = object
+			} else if object = marray[index]; object == nil {
+				return fmt.Errorf("failed to resolve JSON pointer: index '%v' value '%v' was not found", target, pathSeg)
+			}
+		} else {
+			return fmt.Errorf("failed to resolve JSON pointer: index '%v' value '%v' was not found", target, pathSeg)
+		}
+	}
+	return nil
+}
+
+// Object creates a new JSON object at a target path. Returns an error if the
+// path contains a collision with a non object type.
 func (g *Container) Object(path ...string) (*Container, error) {
 	return g.Set(map[string]interface{}{}, path...)
 }
 
-// ObjectP - Does the same as Object, but using a dot notation JSON path.
+// ObjectP creates a new JSON object at a target path using dot notation.
+// Returns an error if the path contains a collision with a non object type.
 func (g *Container) ObjectP(path string) (*Container, error) {
 	return g.Object(strings.Split(path, ".")...)
 }
 
-// ObjectI - Create a new JSON object at an array index. Returns an error if the object is not an
-// array or the index is out of bounds.
+// ObjectI creates a new JSON object at an array index. Returns an error if the
+// object is not an array or the index is out of bounds.
 func (g *Container) ObjectI(index int) (*Container, error) {
 	return g.SetIndex(map[string]interface{}{}, index)
 }
 
-// Array - Create a new JSON array at a path. Returns an error if the path contains a collision with
-// a non object type.
+// Array creates a new JSON array at a path. Returns an error if the path
+// contains a collision with a non object type.
 func (g *Container) Array(path ...string) (*Container, error) {
 	return g.Set([]interface{}{}, path...)
 }
 
-// ArrayP - Does the same as Array, but using a dot notation JSON path.
+// ArrayP creates a new JSON array at a path using dot notation. Returns an
+// error if the path contains a collision with a non object type.
 func (g *Container) ArrayP(path string) (*Container, error) {
 	return g.Array(strings.Split(path, ".")...)
 }
 
-// ArrayI - Create a new JSON array at an array index. Returns an error if the object is not an
-// array or the index is out of bounds.
+// ArrayI creates a new JSON array within an array at an index. Returns an error
+// if the element is not an array or the index is out of bounds.
 func (g *Container) ArrayI(index int) (*Container, error) {
 	return g.SetIndex([]interface{}{}, index)
 }
 
-// ArrayOfSize - Create a new JSON array of a particular size at a path. Returns an error if the
-// path contains a collision with a non object type.
+// ArrayOfSize creates a new JSON array of a particular size at a path. Returns
+// an error if the path contains a collision with a non object type.
 func (g *Container) ArrayOfSize(size int, path ...string) (*Container, error) {
 	a := make([]interface{}, size)
 	return g.Set(a, path...)
 }
 
-// ArrayOfSizeP - Does the same as ArrayOfSize, but using a dot notation JSON path.
+// ArrayOfSizeP creates a new JSON array of a particular size at a path using
+// dot notation. Returns an error if the path contains a collision with a non
+// object type.
 func (g *Container) ArrayOfSizeP(size int, path string) (*Container, error) {
 	return g.ArrayOfSize(size, strings.Split(path, ".")...)
 }
 
-// ArrayOfSizeI - Create a new JSON array of a particular size at an array index. Returns an error
-// if the object is not an array or the index is out of bounds.
+// ArrayOfSizeI create a new JSON array of a particular size within an array at
+// an index. Returns an error if the element is not an array or the index is out
+// of bounds.
 func (g *Container) ArrayOfSizeI(size, index int) (*Container, error) {
 	a := make([]interface{}, size)
 	return g.SetIndex(a, index)
 }
 
-// Delete - Delete an element at a JSON path, an error is returned if the element does not exist.
+// Delete an element at a path, an error is returned if the element does not
+// exist.
 func (g *Container) Delete(path ...string) error {
 	var object interface{}
 
@@ -304,45 +419,40 @@ func (g *Container) Delete(path ...string) error {
 	return nil
 }
 
-// DeleteP - Does the same as Delete, but using a dot notation JSON path.
+// DeleteP deletes an element at a path using dot notation, an error is returned
+// if the element does not exist.
 func (g *Container) DeleteP(path string) error {
 	return g.Delete(strings.Split(path, ".")...)
 }
 
-// Merge - Merges two gabs-containers
-func (g *Container) Merge(toMerge *Container) error {
+// MergeFn merges two objects using a provided function to resolve collisions.
+//
+// The collision function receives two interface{} arguments, destination (the
+// original object) and source (the object being merged into the destination).
+// Which ever value is returned becomes the new value in the destination object
+// at the location of the collision.
+func (g *Container) MergeFn(source *Container, collisionFn func(destination, source interface{}) interface{}) error {
 	var recursiveFnc func(map[string]interface{}, []string) error
 	recursiveFnc = func(mmap map[string]interface{}, path []string) error {
 		for key, value := range mmap {
 			newPath := append(path, key)
 			if g.Exists(newPath...) {
-				target := g.Search(newPath...)
+				existingData := g.Search(newPath...).Data()
 				switch t := value.(type) {
 				case map[string]interface{}:
-					switch targetV := target.Data().(type) {
+					switch existingVal := existingData.(type) {
 					case map[string]interface{}:
 						if err := recursiveFnc(t, newPath); err != nil {
 							return err
 						}
-					case []interface{}:
-						g.Set(append(targetV, t), newPath...)
 					default:
-						newSlice := append([]interface{}{}, targetV)
-						g.Set(append(newSlice, t), newPath...)
-					}
-				case []interface{}:
-					for _, valueOfSlice := range t {
-						if err := g.ArrayAppend(valueOfSlice, newPath...); err != nil {
+						if _, err := g.Set(collisionFn(existingVal, t), newPath...); err != nil {
 							return err
 						}
 					}
 				default:
-					switch targetV := target.Data().(type) {
-					case []interface{}:
-						g.Set(append(targetV, t), newPath...)
-					default:
-						newSlice := append([]interface{}{}, targetV)
-						g.Set(append(newSlice, t), newPath...)
+					if _, err := g.Set(collisionFn(existingData, t), newPath...); err != nil {
+						return err
 					}
 				}
 			} else {
@@ -354,21 +464,48 @@ func (g *Container) Merge(toMerge *Container) error {
 		}
 		return nil
 	}
-	if mmap, ok := toMerge.Data().(map[string]interface{}); ok {
+	if mmap, ok := source.Data().(map[string]interface{}); ok {
 		return recursiveFnc(mmap, []string{})
 	}
 	return nil
 }
 
-//--------------------------------------------------------------------------------------------------
+// Merge a source object into an existing destination object. When a collision
+// is found within the merged structures (both a source and destination object
+// contain the same non-object keys) the result will be an array containing both
+// values, where values that are already arrays will be expanded into the
+// resulting array.
+//
+// It is possible to merge structures will different collision behaviours with
+// MergeFn.
+func (g *Container) Merge(source *Container) error {
+	return g.MergeFn(source, func(dest, source interface{}) interface{} {
+		destArr, destIsArray := dest.([]interface{})
+		sourceArr, sourceIsArray := source.([]interface{})
+		if destIsArray {
+			if sourceIsArray {
+				return append(destArr, sourceArr...)
+			}
+			return append(destArr, source)
+		}
+		if sourceIsArray {
+			return append(append([]interface{}{}, dest), sourceArr...)
+		}
+		return []interface{}{dest, source}
+	})
+}
+
+//------------------------------------------------------------------------------
 
 /*
-Array modification/search - Keeping these options simple right now, no need for anything more
-complicated since you can just cast to []interface{}, modify and then reassign with Set.
+Array modification/search - Keeping these options simple right now, no need for
+anything more complicated since you can just cast to []interface{}, modify and
+then reassign with Set.
 */
 
-// ArrayAppend - Append a value onto a JSON array. If the target is not a JSON array then it will be
-// converted into one, with its contents as the first element of the array.
+// ArrayAppend attempts to append a value onto a JSON array at a path. If the
+// target is not a JSON array then it will be converted into one, with its
+// original contents set to the first element of the array.
 func (g *Container) ArrayAppend(value interface{}, path ...string) error {
 	if array, ok := g.Search(path...).Data().([]interface{}); ok {
 		array = append(array, value)
@@ -386,12 +523,15 @@ func (g *Container) ArrayAppend(value interface{}, path ...string) error {
 	return err
 }
 
-// ArrayAppendP - Append a value onto a JSON array using a dot notation JSON path.
+// ArrayAppendP attempts to append a value onto a JSON array at a path using dot
+// notation. If the target is not a JSON array then it will be converted into
+// one, with its original contents set to the first element of the array.
 func (g *Container) ArrayAppendP(value interface{}, path string) error {
 	return g.ArrayAppend(value, strings.Split(path, ".")...)
 }
 
-// ArrayRemove - Remove an element from a JSON array.
+// ArrayRemove attempts to remove an element identified by an index from a JSON
+// array at a path.
 func (g *Container) ArrayRemove(index int, path ...string) error {
 	if index < 0 {
 		return ErrOutOfBounds
@@ -409,12 +549,14 @@ func (g *Container) ArrayRemove(index int, path ...string) error {
 	return err
 }
 
-// ArrayRemoveP - Remove an element from a JSON array using a dot notation JSON path.
+// ArrayRemoveP attempts to remove an element identified by an index from a JSON
+// array at a path using dot notation.
 func (g *Container) ArrayRemoveP(index int, path string) error {
 	return g.ArrayRemove(index, strings.Split(path, ".")...)
 }
 
-// ArrayElement - Access an element from a JSON array.
+// ArrayElement attempts to access an element by an index from a JSON array at a
+// path.
 func (g *Container) ArrayElement(index int, path ...string) (*Container, error) {
 	if index < 0 {
 		return &Container{nil}, ErrOutOfBounds
@@ -429,12 +571,13 @@ func (g *Container) ArrayElement(index int, path ...string) (*Container, error)
 	return &Container{nil}, ErrOutOfBounds
 }
 
-// ArrayElementP - Access an element from a JSON array using a dot notation JSON path.
+// ArrayElementP attempts to access an element by an index from a JSON array at
+// a path using dot notation.
 func (g *Container) ArrayElementP(index int, path string) (*Container, error) {
 	return g.ArrayElement(index, strings.Split(path, ".")...)
 }
 
-// ArrayCount - Count the number of elements in a JSON array.
+// ArrayCount counts the number of elements in a JSON array at a path.
 func (g *Container) ArrayCount(path ...string) (int, error) {
 	if array, ok := g.Search(path...).Data().([]interface{}); ok {
 		return len(array), nil
@@ -442,14 +585,15 @@ func (g *Container) ArrayCount(path ...string) (int, error) {
 	return 0, ErrNotArray
 }
 
-// ArrayCountP - Count the number of elements in a JSON array using a dot notation JSON path.
+// ArrayCountP counts the number of elements in a JSON array at a path using dot
+// notation.
 func (g *Container) ArrayCountP(path string) (int, error) {
 	return g.ArrayCount(strings.Split(path, ".")...)
 }
 
-//--------------------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
 
-// Bytes - Converts the contained object back to a JSON []byte blob.
+// Bytes marshals an element to a JSON []byte blob.
 func (g *Container) Bytes() []byte {
 	if g.Data() != nil {
 		if bytes, err := json.Marshal(g.object); err == nil {
@@ -459,7 +603,8 @@ func (g *Container) Bytes() []byte {
 	return []byte("{}")
 }
 
-// BytesIndent - Converts the contained object to a JSON []byte blob formatted with prefix, indent.
+// BytesIndent marshals an element to a JSON []byte blob formatted with a prefix
+// and indent string.
 func (g *Container) BytesIndent(prefix string, indent string) []byte {
 	if g.object != nil {
 		if bytes, err := json.MarshalIndent(g.object, prefix, indent); err == nil {
@@ -469,12 +614,13 @@ func (g *Container) BytesIndent(prefix string, indent string) []byte {
 	return []byte("{}")
 }
 
-// String - Converts the contained object to a JSON formatted string.
+// String marshals an element to a JSON formatted string.
 func (g *Container) String() string {
 	return string(g.Bytes())
 }
 
-// StringIndent - Converts the contained object back to a JSON formatted string with prefix, indent.
+// StringIndent marshals an element to a JSON string formatted with a prefix and
+// indent string.
 func (g *Container) StringIndent(prefix string, indent string) string {
 	return string(g.BytesIndent(prefix, indent))
 }
@@ -496,10 +642,9 @@ func EncodeOptIndent(prefix string, indent string) EncodeOpt {
 	}
 }
 
-// EncodeJSON - Encodes the contained object back to a JSON formatted []byte
-// using a variant list of modifier functions for the encoder being used.
-// Functions for modifying the output are prefixed with EncodeOpt, e.g.
-// EncodeOptHTMLEscape.
+// EncodeJSON marshals an element to a JSON formatted []byte using a variant
+// list of modifier functions for the encoder being used. Functions for
+// modifying the output are prefixed with EncodeOpt, e.g. EncodeOptHTMLEscape.
 func (g *Container) EncodeJSON(encodeOpts ...EncodeOpt) []byte {
 	var b bytes.Buffer
 	encoder := json.NewEncoder(&b)
@@ -517,17 +662,18 @@ func (g *Container) EncodeJSON(encodeOpts ...EncodeOpt) []byte {
 	return result
 }
 
-// New - Create a new gabs JSON object.
+// New creates a new gabs JSON object.
 func New() *Container {
 	return &Container{map[string]interface{}{}}
 }
 
-// Consume - Gobble up an already converted JSON object, or a fresh map[string]interface{} object.
+// Consume an already unmarshalled JSON object (or a new map[string]interface{})
+// into a *Container.
 func Consume(root interface{}) (*Container, error) {
 	return &Container{root}, nil
 }
 
-// ParseJSON - Convert a string into a representation of the parsed JSON.
+// ParseJSON unmarshals a JSON byte slice into a *Container.
 func ParseJSON(sample []byte) (*Container, error) {
 	var gabs Container
 
@@ -538,7 +684,7 @@ func ParseJSON(sample []byte) (*Container, error) {
 	return &gabs, nil
 }
 
-// ParseJSONDecoder - Convert a json.Decoder into a representation of the parsed JSON.
+// ParseJSONDecoder applies a json.Decoder to a *Container.
 func ParseJSONDecoder(decoder *json.Decoder) (*Container, error) {
 	var gabs Container
 
@@ -549,7 +695,7 @@ func ParseJSONDecoder(decoder *json.Decoder) (*Container, error) {
 	return &gabs, nil
 }
 
-// ParseJSONFile - Read a file and convert into a representation of the parsed JSON.
+// ParseJSONFile reads a file and unmarshals the contents into a *Container.
 func ParseJSONFile(path string) (*Container, error) {
 	if len(path) > 0 {
 		cBytes, err := ioutil.ReadFile(path)
@@ -567,7 +713,7 @@ func ParseJSONFile(path string) (*Container, error) {
 	return nil, ErrInvalidPath
 }
 
-// ParseJSONBuffer - Read the contents of a buffer into a representation of the parsed JSON.
+// ParseJSONBuffer reads a buffer and unmarshals the contents into a *Container.
 func ParseJSONBuffer(buffer io.Reader) (*Container, error) {
 	var gabs Container
 	jsonDecoder := json.NewDecoder(buffer)
@@ -578,4 +724,4 @@ func ParseJSONBuffer(buffer io.Reader) (*Container, error) {
 	return &gabs, nil
 }
 
-//--------------------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

vendor/github.com/Jeffail/gabs/go.mod

@@ -0,0 +1 @@
+module github.com/Jeffail/gabs

vendor/github.com/Philipp15b/go-steam/.gitmodules

@@ -1,3 +1,6 @@
 [submodule "generator/SteamKit"]
 	path = generator/SteamKit
-	url = https://github.com/Philipp15b/SteamKit.git
+	url = https://github.com/SteamRE/SteamKit.git
+[submodule "generator/Protobufs"]
+	path = generator/Protobufs
+	url = https://github.com/SteamDatabase/Protobufs.git

vendor/github.com/Philipp15b/go-steam/auth.go

@@ -119,7 +119,7 @@ func (a *Auth) handleLogOnResponse(packet *Packet) {
 			ExtendedResult:            EResult(body.GetEresultExtended()),
 			OutOfGameSecsPerHeartbeat: body.GetOutOfGameHeartbeatSeconds(),
 			InGameSecsPerHeartbeat:    body.GetInGameHeartbeatSeconds(),
-			PublicIp:                  body.GetPublicIp(),
+			PublicIp:                  body.GetDeprecatedPublicIp(),
 			ServerTime:                body.GetRtime32ServerTime(),
 			AccountFlags:              EAccountFlags(body.GetAccountFlags()),
 			ClientSteamId:             SteamId(body.GetClientSuppliedSteamid()),
@@ -127,7 +127,7 @@ func (a *Auth) handleLogOnResponse(packet *Packet) {
 			CellId:                    body.GetCellId(),
 			CellIdPingThreshold:       body.GetCellIdPingThreshold(),
 			Steam2Ticket:              body.GetSteam2Ticket(),
-			UsePics:                   body.GetUsePics(),
+			UsePics:                   body.GetDeprecatedUsePics(),
 			WebApiUserNonce:           body.GetWebapiAuthenticateUserNonce(),
 			IpCountryCode:             body.GetIpCountryCode(),
 			VanityUrl:                 body.GetVanityUrl(),

vendor/github.com/Philipp15b/go-steam/go.mod

@@ -0,0 +1,8 @@
+module github.com/Philipp15b/go-steam
+
+go 1.14
+
+require (
+	github.com/davecgh/go-spew v1.1.1
+	github.com/golang/protobuf v1.4.2
+)

vendor/github.com/Philipp15b/go-steam/go.sum

@@ -0,0 +1,22 @@
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=