Release v1.22.0 (#1385)

* Add vk bridge

* Vk bridge attachments

* Vk bridge forwarded messages

* Vk bridge sample config and code cleanup

* Vk bridge add vendor

* Vk bridge message edit

* Vk bridge: fix fetching names of other bots

* Vk bridge: code cleanup

* Vk bridge: fix shadows declaration

* Vk bridge: remove UseFileURL

.dockerignore

@@ -0,0 +1,2 @@
+Dockerfile
+tgs.Dockerfile

.github/ISSUE_TEMPLATE/Bug_report.md

@@ -1,6 +1,7 @@
 ---
 name: Bug report
 about: Create a report to help us improve. (Check the FAQ on the wiki first)
+labels: bug
 
 ---
 

.github/ISSUE_TEMPLATE/Feature_request.md

@@ -1,6 +1,7 @@
 ---
 name: Feature request
 about: Suggest an idea for this project
+labels: enhancement
 
 ---
 

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,71 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+name: "CodeQL"
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [master]
+  schedule:
+    - cron: '0 16 * * 1'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        # Override automatic language detection by changing the below list
+        # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python']
+        language: ['go']
+        # Learn more...
+        # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+      with:
+        # We must fetch at least the immediate parents so that if this is
+        # a pull request then we can checkout the head.
+        fetch-depth: 2
+
+    # If this run was triggered by a pull request event, then checkout
+    # the head of the pull request instead of the merge commit.
+    - run: git checkout HEAD^2
+      if: ${{ github.event_name == 'pull_request' }}
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file. 
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/development.yml

@@ -0,0 +1,57 @@
+name: Development
+on: [push, pull_request]
+jobs:
+  lint:
+    name: golangci-lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 20
+      - name: Run golangci-lint
+        uses: golangci/golangci-lint-action@v2
+        with:
+          version: v1.29
+          args: "-v --new-from-rev HEAD~5"
+  test-build-upload:
+    strategy:
+      matrix:
+        go-version: [1.14.x, 1.15.x]
+        platform: [ubuntu-latest]
+    runs-on: ${{ matrix.platform }}
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+    - name: Checkout code
+      uses: actions/checkout@v2
+      with:
+          fetch-depth: 0
+    - name: Test
+      run: go test ./... -mod=vendor
+    - name: Build
+      run: |
+        mkdir -p output/{win,lin,arm,mac}
+        VERSION=$(git describe --tags)
+        GOOS=linux GOARCH=amd64 go build -mod=vendor -ldflags "-s -X main.githash=$(git log --pretty=format:'%h' -n 1)" -o output/lin/matterbridge-$VERSION-linux-amd64
+        GOOS=windows GOARCH=amd64 go build -mod=vendor -ldflags "-s -X main.githash=$(git log --pretty=format:'%h' -n 1)" -o output/win/matterbridge-$VERSION-windows-amd64.exe
+        GOOS=darwin GOARCH=amd64 go build -mod=vendor -ldflags "-s -X main.githash=$(git log --pretty=format:'%h' -n 1)" -o output/mac/matterbridge-$VERSION-darwin-amd64
+    - name: Upload linux 64-bit
+      if: startsWith(matrix.go-version,'1.15')
+      uses: actions/upload-artifact@v2
+      with:
+        name: matterbridge-linux-64bit
+        path: output/lin
+    - name: Upload windows 64-bit
+      if: startsWith(matrix.go-version,'1.15')
+      uses: actions/upload-artifact@v2
+      with:
+        name: matterbridge-windows-64bit
+        path: output/win
+    - name: Upload darwin 64-bit
+      if: startsWith(matrix.go-version,'1.15')
+      uses: actions/upload-artifact@v2
+      with:
+        name: matterbridge-darwin-64bit
+        path: output/mac

.golangci.yaml

@@ -23,7 +23,7 @@ run:
   # default value is empty list, but next dirs are always skipped independently
   # from this option's value:
   #   	vendor$, third_party$, testdata$, examples$, Godeps$, builtin$
-  skip-dirs:
+  skip-dirs: gateway/bridgemap$
 
   # which files to skip: they will be analyzed, but issues from them
   # won't be reported. Default value is empty list, but there is
@@ -91,7 +91,6 @@ linters-settings:
     # Correct spellings using locale preferences for US or UK.
     # Default is to use a neutral variety of English.
     # Setting locale to US will correct the British spelling of 'colour' to 'color'.
-    locale: US
   lll:
     # max line length, lines longer will be reported. Default is 120.
     # '\t' is counted as 1 character by default, and can be changed with the tab-width option
@@ -175,7 +174,14 @@ linters:
     - maligned
     - prealloc
     - wsl
-
+    - gomnd
+    - godox
+    - goerr113
+    - testpackage
+    - godot
+    - interfacer
+    - goheader
+    - noctx
 
 # rules to deal with reported isues
 issues:

.travis.yml

@@ -1,56 +0,0 @@
-language: go
-go_import_path: github.com/42wim/matterbridge
-
-# We have everything vendored so this helps TravisCI not run `go get ...`.
-install: true
-
-git:
-  depth: 200
-
-notifications:
-  email: false
-
-branches:
-  only:
-  - master
-  - /.*/
-
-jobs:
-  include:
-  - stage: lint
-    # Run linting in one Go environment only.
-    script: ./ci/lint.sh
-    go: 1.13.x
-    env:
-    - GO111MODULE=on
-    - GOLANGCI_VERSION="v1.21.0"
-  - stage: test
-    # Run tests in a combination of Go environments.
-    script: ./ci/test.sh
-    go: 1.12.x
-    env:
-    - GO111MODULE=off
-  - script: ./ci/test.sh
-    go: 1.12.x
-    env:
-    - GO111MODULE=on
-  - script: ./ci/test.sh
-    go: 1.13.x
-    env:
-    - GO111MODULE=on
-    - REPORT_COVERAGE=1
-    - BINDEPLOY=1
-
-before_deploy: /bin/bash ci/bintray.sh
-
-deploy:
-  on:
-    all_branches: true
-    condition: $BINDEPLOY = 1
-  provider: bintray
-  edge:
-    branch: v1.8.47
-  file: ci/deploy.json
-  user: 42wim
-  key:
-    secure: "CeXXe6JOmt7HYR81MdWLua0ltQHhDdkIeRGBFbgd7hkb1wi8eF9DgpAcQrTso8NIlHNZmSAP46uhFgsRvkuezzX0ygalZ7DCJyAyn3sAMEh+UQSHV1WGThRehTtidqRGjetzsIGSwdrJOWil+XTfbO1Z8DGzfakhSuAZka8CM4BAoe3YeP9rYK8h+84x0GHfczvsLtXZ3mWLvQuwe4pK6+ItBCUg0ae7O7ZUpWHy0xQQkkWztY/6RAzXfaG7DuGjIw+20fhx3WOXRNpHCtZ6Bc3qERCpk0s1HhlQWlrN9wDaFTBWYwlvSnNgvxxMbNXJ6RrRJ0l0bA7FUswYwyroxhzrGLdzWDg8dHaQkypocngdalfhpsnoO9j3ApJhomUFJ3UoEq5nOGRUrKn8MPi+dP0zE4kNQ3e4VNa1ufNrvfpWolMg3xh8OXuhQdD5wIM5zFAbRJLqWSCVAjPq4DDPecmvXBOlIial7oa312lN5qnBnUjvAcxszZ+FUyDHT1Grxzna4tMwxY9obPzZUzm7359AOCCwIQFVB8GLqD2nwIstcXS0zGRz+fhviPipHuBa02q5bGUZwmkvrSNab0s8Jo7pCrel2Rz3nWPKaiCfq2WjbW1CLheSMkOQrjsdUd1hhbqNWFPUjJPInTc77NAKCfm5runv5uyowRLh4NNd0sI="

Dockerfile

@@ -1,11 +1,14 @@
-FROM alpine:edge
+FROM alpine AS builder
-ENTRYPOINT ["/bin/matterbridge"]
 
-COPY . /go/src/github.com/42wim/matterbridge
+COPY . /go/src/matterbridge
-RUN apk update && apk add go git gcc musl-dev ca-certificates mailcap \
+RUN apk --no-cache add go git \
-        && cd /go/src/github.com/42wim/matterbridge \
+        && cd /go/src/matterbridge \
-        && export GOPATH=/go \
+        && go build -mod vendor -ldflags "-X main.githash=$(git log --pretty=format:'%h' -n 1)" -o /bin/matterbridge
-        && go get \
+
-        && go build -x -ldflags "-X main.githash=$(git log --pretty=format:'%h' -n 1)" -o /bin/matterbridge \
+FROM alpine
-        && rm -rf /go \
+RUN apk --no-cache add ca-certificates mailcap
-        && apk del --purge git go gcc musl-dev
+COPY --from=builder /bin/matterbridge /bin/matterbridge
+RUN mkdir /etc/matterbridge \
+  && touch /etc/matterbridge/matterbridge.toml \
+  && ln -sf /matterbridge.toml /etc/matterbridge/matterbridge.toml
+ENTRYPOINT ["/bin/matterbridge", "-conf", "/etc/matterbridge/matterbridge.toml"]

README.md

@@ -9,26 +9,26 @@ Letting people be where they want to be.<br />
 
    <sup>
 
+[Discord][mb-discord] |
 [Gitter][mb-gitter] |
 [IRC][mb-irc] |
-[Discord][mb-discord] |
+[Keybase][mb-keybase] |
 [Matrix][mb-matrix] |
-[Slack][mb-slack] |
 [Mattermost][mb-mattermost] |
+[MSTeams][mb-msteams] |
 [Rocket.Chat][mb-rocketchat] |
-[XMPP][mb-xmpp] |
+[Slack][mb-slack] |
+[Telegram][mb-telegram] |
 [Twitch][mb-twitch] |
 [WhatsApp][mb-whatsapp] |
+[XMPP][mb-xmpp] |
 [Zulip][mb-zulip] |
-[Telegram][mb-telegram] |
-[Keybase][mb-keybase] |
 And more...
 </sup>
 
 ---
 
 [![Download stable](https://img.shields.io/github/release/42wim/matterbridge.svg?label=download%20stable)](https://github.com/42wim/matterbridge/releases/latest)
-[![Download dev](https://img.shields.io/bintray/v/42wim/nightly/Matterbridge.svg?label=download%20dev&colorB=007ec6)](https://bintray.com/42wim/nightly/Matterbridge/_latestVersion)
 [![Maintainability](https://api.codeclimate.com/v1/badges/82dff70ef2ba85a6173a/maintainability)](https://codeclimate.com/github/42wim/matterbridge/maintainability)
 [![Test Coverage](https://api.codeclimate.com/v1/badges/82dff70ef2ba85a6173a/test_coverage)](https://codeclimate.com/github/42wim/matterbridge/test_coverage)<br />
 
@@ -70,7 +70,7 @@ And more...
   - [Changelog](#changelog)
   - [FAQ](#faq)
   - [Related projects](#related-projects)
-  - [Articles](#articles)
+  - [Articles / Tutorials](#articles--tutorials)
   - [Thanks](#thanks)
 
 ## Features
@@ -86,29 +86,37 @@ And more...
 
 ### Natively supported
 
-- [Mattermost](https://github.com/mattermost/mattermost-server/) 4.x, 5.x
-- [IRC](http://www.mirc.com/servers.html)
-- [XMPP](https://xmpp.org)
-- [Gitter](https://gitter.im)
-- [Slack](https://slack.com)
 - [Discord](https://discordapp.com)
-- [Telegram](https://telegram.org)
+- [Gitter](https://gitter.im)
-- [Rocket.chat](https://rocket.chat)
+- [IRC](http://www.mirc.com/servers.html)
-- [Matrix](https://matrix.org)
-- [Steam](https://store.steampowered.com/)
-- [Twitch](https://twitch.tv)
-- [Ssh-chat](https://github.com/shazow/ssh-chat)
-- [WhatsApp](https://www.whatsapp.com/)
-- [Zulip](https://zulipchat.com)
 - [Keybase](https://keybase.io)
+- [Matrix](https://matrix.org)
+- [Mattermost](https://github.com/mattermost/mattermost-server/)
+- [Microsoft Teams](https://teams.microsoft.com)
+- [Mumble](https://www.mumble.info/)
+- [Nextcloud Talk](https://nextcloud.com/talk/)
+- [Rocket.chat](https://rocket.chat)
+- [Slack](https://slack.com)
+- [Ssh-chat](https://github.com/shazow/ssh-chat)
+- ~~[Steam](https://store.steampowered.com/)~~
+  - Not supported anymore, see [here](https://github.com/Philipp15b/go-steam/issues/94) for more info.
+- [Telegram](https://telegram.org)
+- [Twitch](https://twitch.tv)
+- [VK](https://vk.com/)
+- [WhatsApp](https://www.whatsapp.com/)
+- [XMPP](https://xmpp.org)
+- [Zulip](https://zulipchat.com)
 
 ### 3rd party via matterbridge api
 
-- [Minecraft](https://github.com/elytra/MatterLink)
-- [Reddit](https://github.com/bonehurtingjuice/mattereddit)
-- [Facebook messenger](https://github.com/VictorNine/fbridge)
 - [Discourse](https://github.com/DeclanHoare/matterbabble)
+- [Facebook messenger](https://github.com/VictorNine/fbridge)
+- [Minecraft](https://github.com/elytra/MatterLink)
+- [Minecraft](https://github.com/raws/mattercraft)
+- [Reddit](https://github.com/bonehurtingjuice/mattereddit)
 - [Counter-Strike, half-life and more](https://forums.alliedmods.net/showthread.php?t=319430)
+- [MatterAMXX](https://github.com/GabeIggy/MatterAMXX)
+- [Vintage Story](https://github.com/NikkyAI/vs-matterbridge)
 
 ### API
 
@@ -118,60 +126,63 @@ More info and examples on the [wiki](https://github.com/42wim/matterbridge/wiki/
 Used by the projects below. Feel free to make a PR to add your project to this list.
 
 - [MatterLink](https://github.com/elytra/MatterLink) (Matterbridge link for Minecraft Server chat)
+- [Minecraft](https://github.com/raws/mattercraft) (Matterbridge link for Minecraft Server chat)
 - [pyCord](https://github.com/NikkyAI/pyCord) (crossplatform chatbot)
 - [Mattereddit](https://github.com/bonehurtingjuice/mattereddit) (Reddit chat support)
 - [fbridge](https://github.com/VictorNine/fbridge) (Facebook messenger support)
 - [matterbabble](https://github.com/DeclanHoare/matterbabble) (Discourse support)
 - [MatterAMXX](https://forums.alliedmods.net/showthread.php?t=319430) (Counter-Strike, half-life and more via AMXX mod)
+- [Vintage Story](https://github.com/NikkyAI/vs-matterbridge)
 
 ## Chat with us
 
 Questions or want to test on your favorite platform? Join below:
 
+- [Discord][mb-discord]
 - [Gitter][mb-gitter]
 - [IRC][mb-irc]
-- [Discord][mb-discord]
+- [Keybase][mb-keybase]
 - [Matrix][mb-matrix]
-- [Slack][mb-slack]
 - [Mattermost][mb-mattermost]
 - [Rocket.Chat][mb-rocketchat]
-- [XMPP][mb-xmpp] (matterbridge@conference.jabber.de)
+- [Slack][mb-slack]
-- [Twitch][mb-twitch]
-- [Zulip][mb-zulip]
 - [Telegram][mb-telegram]
+- [Twitch][mb-twitch]
+- [XMPP][mb-xmpp] (matterbridge@conference.jabber.de)
+- [Zulip][mb-zulip]
 
 ## Screenshots
 
-See https://github.com/42wim/matterbridge/wiki
+See <https://github.com/42wim/matterbridge/wiki>
 
 ## Installing / upgrading
 
 ### Binaries
 
-- Latest stable release [v1.16.5](https://github.com/42wim/matterbridge/releases/latest)
+- Latest stable release [v1.22.0](https://github.com/42wim/matterbridge/releases/latest)
-- Development releases (follows master) can be downloaded [here](https://dl.bintray.com/42wim/nightly/)
+- Development releases (follows master) can be downloaded [here](https://github.com/42wim/matterbridge/actions) selecting the latest green build and then artifacts.
 
-To install or upgrade just download the latest [binary](https://github.com/42wim/matterbridge/releases/latest) and follow the instructions on the [howto](https://github.com/42wim/matterbridge/wiki/How-to-create-your-config) for a step by step walkthrough for creating your configuration.
+To install or upgrade just download the latest [binary](https://github.com/42wim/matterbridge/releases/latest). On \*nix platforms you may need to make the binary executable - you can do this by running `chmod a+x` on the binary (example: `chmod a+x matterbridge-1.20.0-linux-64bit`). After downloading (and making the binary executable, if necessary), follow the instructions on the [howto](https://github.com/42wim/matterbridge/wiki/How-to-create-your-config) for a step by step walkthrough for creating your configuration.
 
 ### Packages
 
 - [Overview](https://repology.org/metapackage/matterbridge/versions)
+- [snap](https://snapcraft.io/matterbridge)
 
 ## Building
 
 Most people just want to use binaries, you can find those [here](https://github.com/42wim/matterbridge/releases/latest)
 
 If you really want to build from source, follow these instructions:
-Go 1.12+ is required. Make sure you have [Go](https://golang.org/doc/install) properly installed.
+Go 1.13+ is required. Make sure you have [Go](https://golang.org/doc/install) properly installed.
 
-
+```bash
-```
 go get github.com/42wim/matterbridge
 ```
 
 You should now have matterbridge binary in the ~/go/bin directory:
 
-```
+```bash
 $ ls ~/go/bin/
 matterbridge
 ```
@@ -254,7 +265,7 @@ RemoteNickFormat="[{PROTOCOL}/{BRIDGE}] <{NICK}> "
 
 See [howto](https://github.com/42wim/matterbridge/wiki/How-to-create-your-config) for a step by step walkthrough for creating your configuration.
 
-```
+```bash
 Usage of ./matterbridge:
   -conf string
         config file (default "matterbridge.toml")
@@ -291,18 +302,23 @@ See [FAQ](https://github.com/42wim/matterbridge/wiki/FAQ)
 - [fbridge](https://github.com/VictorNine/fbridge) (Facebook messenger support)
 - [isla](https://github.com/alphachung/isla) (Bot for Discord-Telegram groups used alongside matterbridge)
 - [matterbabble](https://github.com/DeclanHoare/matterbabble) (Connect Discourse threads to Matterbridge)
+- [nextcloud talk](https://github.com/nextcloud/talk_matterbridge) (Integrates matterbridge in Nextcloud Talk)
+- [mattercraft](https://github.com/raws/mattercraft) (Minecraft bridge)
+- [vs-matterbridge](https://github.com/NikkyAI/vs-matterbridge) (Vintage Story bridge)
 
-## Articles
+## Articles / Tutorials
 
 - [matterbridge on kubernetes](https://medium.freecodecamp.org/using-kubernetes-to-deploy-a-chat-gateway-or-when-technology-works-like-its-supposed-to-a169a8cd69a3)
-- https://mattermost.com/blog/connect-irc-to-mattermost/
+- <https://mattermost.com/blog/connect-irc-to-mattermost/>
-- https://blog.valvin.fr/2016/09/17/mattermost-et-un-channel-irc-cest-possible/
+- <https://blog.valvin.fr/2016/09/17/mattermost-et-un-channel-irc-cest-possible/>
-- https://blog.brightscout.com/top-10-mattermost-integrations/
+- <https://blog.brightscout.com/top-10-mattermost-integrations/>
-- http://bencey.co.nz/2018/09/17/bridge/
+- <https://www.algoo.fr/blog/2018/01/19/recouvrez-votre-liberte-en-quittant-slack-pour-un-mattermost-auto-heberge/>
-- https://www.algoo.fr/blog/2018/01/19/recouvrez-votre-liberte-en-quittant-slack-pour-un-mattermost-auto-heberge/
+- <https://kopano.com/blog/matterbridge-bridging-mattermost-chat/>
-- https://kopano.com/blog/matterbridge-bridging-mattermost-chat/
+- <https://www.stitcher.com/s/?eid=52382713>
-- https://www.stitcher.com/s/?eid=52382713
+- <https://daniele.tech/2019/02/how-to-use-matterbridge-to-connect-2-different-slack-workspaces/>
-- https://daniele.tech/2019/02/how-to-use-matterbridge-to-connect-2-different-slack-workspaces/
+- <https://userlinux.net/mattermost-and-matterbridge.html>
+- <https://nextcloud.com/blog/bridging-chat-services-in-talk/>
+- Youtube: [whatsapp - telegram bridging](https://www.youtube.com/watch?v=W-VXISoKtNc)
 
 ## Thanks
 
@@ -315,36 +331,42 @@ See [FAQ](https://github.com/42wim/matterbridge/wiki/FAQ)
 
 Matterbridge wouldn't exist without these libraries:
 
-- discord - https://github.com/bwmarrin/discordgo
+- discord - <https://github.com/bwmarrin/discordgo>
-- echo - https://github.com/labstack/echo
+- echo - <https://github.com/labstack/echo>
-- gitter - https://github.com/sromku/go-gitter
+- gitter - <https://github.com/sromku/go-gitter>
-- gops - https://github.com/google/gops
+- gops - <https://github.com/google/gops>
-- gozulipbot - https://github.com/ifo/gozulipbot
+- gozulipbot - <https://github.com/ifo/gozulipbot>
-- irc - https://github.com/lrstanley/girc
+- gumble - <https://github.com/layeh/gumble>
-- mattermost - https://github.com/mattermost/mattermost-server
+- irc - <https://github.com/lrstanley/girc>
-- matrix - https://github.com/matrix-org/gomatrix
+- keybase - <https://github.com/keybase/go-keybase-chat-bot>
-- sshchat - https://github.com/shazow/ssh-chat
+- matrix - <https://github.com/matrix-org/gomatrix>
-- slack - https://github.com/nlopes/slack
+- mattermost - <https://github.com/mattermost/mattermost-server>
-- steam - https://github.com/Philipp15b/go-steam
+- msgraph.go - <https://github.com/yaegashi/msgraph.go>
-- telegram - https://github.com/go-telegram-bot-api/telegram-bot-api
+- mumble - <https://github.com/layeh/gumble>
-- xmpp - https://github.com/mattn/go-xmpp
+- nctalk - <https://github.com/gary-kim/go-nc-talk>
-- whatsapp - https://github.com/Rhymen/go-whatsapp/
+- slack - <https://github.com/nlopes/slack>
-- zulip - https://github.com/ifo/gozulipbot
+- sshchat - <https://github.com/shazow/ssh-chat>
-- tengo - https://github.com/d5/tengo
+- steam - <https://github.com/Philipp15b/go-steam>
-- keybase - https://github.com/keybase/go-keybase-chat-bot
+- telegram - <https://github.com/go-telegram-bot-api/telegram-bot-api>
+- tengo - <https://github.com/d5/tengo>
+- vk - <https://github.com/SevereCloud/vksdk>
+- whatsapp - <https://github.com/Rhymen/go-whatsapp>
+- xmpp - <https://github.com/mattn/go-xmpp>
+- zulip - <https://github.com/ifo/gozulipbot>
 
 <!-- Links -->
 
+[mb-discord]: https://discord.gg/AkKPtrQ
 [mb-gitter]: https://gitter.im/42wim/matterbridge
 [mb-irc]: https://webchat.freenode.net/?channels=matterbridgechat
-[mb-discord]: https://discord.gg/AkKPtrQ
+[mb-keybase]: https://keybase.io/team/matterbridge
 [mb-matrix]: https://riot.im/app/#/room/#matterbridge:matrix.org
-[mb-slack]: https://join.slack.com/matterbridgechat/shared_invite/MjEwODMxNjU1NDMwLTE0OTk2MTU3NTMtMzZkZmRiNDZhOA
 [mb-mattermost]: https://framateam.org/signup_user_complete/?id=tfqm33ggop8x3qgu4boeieta6e
+[mb-msteams]: https://teams.microsoft.com/join/hj92x75gd3y7
 [mb-rocketchat]: https://open.rocket.chat/channel/matterbridge
-[mb-xmpp]: https://inverse.chat/
+[mb-slack]: https://join.slack.com/t/matterbridgechat/shared_invite/zt-2ourq2h2-7YvyYBq2WFGC~~zEzA68_Q
+[mb-telegram]: https://t.me/Matterbridge
 [mb-twitch]: https://www.twitch.tv/matterbridge
 [mb-whatsapp]: https://www.whatsapp.com/
-[mb-keybase]: https://keybase.io
+[mb-xmpp]: https://inverse.chat/
 [mb-zulip]: https://matterbridge.zulipchat.com/register/
-[mb-telegram]: https://t.me/Matterbridge

bridge/api/api.go

@@ -6,17 +6,20 @@ import (
 	"sync"
 	"time"
 
+	"gopkg.in/olahol/melody.v1"
+
 	"github.com/42wim/matterbridge/bridge"
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/labstack/echo/v4"
 	"github.com/labstack/echo/v4/middleware"
-	"github.com/zfjagann/golang-ring"
+	ring "github.com/zfjagann/golang-ring"
 )
 
 type API struct {
 	Messages ring.Ring
 	sync.RWMutex
 	*bridge.Config
+	mrouter *melody.Melody
 }
 
 type Message struct {
@@ -32,6 +35,32 @@ func New(cfg *bridge.Config) bridge.Bridger {
 	e := echo.New()
 	e.HideBanner = true
 	e.HidePort = true
+
+	b.mrouter = melody.New()
+	b.mrouter.HandleMessage(func(s *melody.Session, msg []byte) {
+		message := config.Message{}
+		err := json.Unmarshal(msg, &message)
+		if err != nil {
+			b.Log.Errorf("failed to decode message from byte[] '%s'", string(msg))
+			return
+		}
+		b.handleWebsocketMessage(message, s)
+	})
+	b.mrouter.HandleConnect(func(session *melody.Session) {
+		greet := b.getGreeting()
+		data, err := json.Marshal(greet)
+		if err != nil {
+			b.Log.Errorf("failed to encode message '%v'", greet)
+			return
+		}
+		err = session.Write(data)
+		if err != nil {
+			b.Log.Errorf("failed to write message '%s'", string(data))
+			return
+		}
+		// TODO: send message history buffer from `b.Messages` here
+	})
+
 	b.Messages = ring.Ring{}
 	if b.GetInt("Buffer") != 0 {
 		b.Messages.SetCapacity(b.GetInt("Buffer"))
@@ -41,9 +70,17 @@ func New(cfg *bridge.Config) bridge.Bridger {
 			return key == b.GetString("Token"), nil
 		}))
 	}
+
+	// Set RemoteNickFormat to a sane default
+	if !b.IsKeySet("RemoteNickFormat") {
+		b.Log.Debugln("RemoteNickFormat is unset, defaulting to \"{NICK}\"")
+		b.Config.Config.Viper().Set(b.GetConfigKey("RemoteNickFormat"), "{NICK}")
+	}
+
 	e.GET("/api/health", b.handleHealthcheck)
 	e.GET("/api/messages", b.handleMessages)
 	e.GET("/api/stream", b.handleStream)
+	e.GET("/api/websocket", b.handleWebsocket)
 	e.POST("/api/message", b.handlePostMessage)
 	go func() {
 		if b.GetString("BindAddress") == "" {
@@ -58,13 +95,13 @@ func New(cfg *bridge.Config) bridge.Bridger {
 func (b *API) Connect() error {
 	return nil
 }
+
 func (b *API) Disconnect() error {
 	return nil
-
 }
+
 func (b *API) JoinChannel(channel config.ChannelInfo) error {
 	return nil
-
 }
 
 func (b *API) Send(msg config.Message) (string, error) {
@@ -74,7 +111,14 @@ func (b *API) Send(msg config.Message) (string, error) {
 	if msg.Event == config.EventMsgDelete {
 		return "", nil
 	}
-	b.Messages.Enqueue(&msg)
+	b.Log.Debugf("enqueueing message from %s on ring buffer", msg.Username)
+	b.Messages.Enqueue(msg)
+
+	data, err := json.Marshal(msg)
+	if err != nil {
+		b.Log.Errorf("failed to encode message  '%s'", msg)
+	}
+	_ = b.mrouter.Broadcast(data)
 	return "", nil
 }
 
@@ -106,18 +150,23 @@ func (b *API) handleMessages(c echo.Context) error {
 	return nil
 }
 
-func (b *API) handleStream(c echo.Context) error {
+func (b *API) getGreeting() config.Message {
-	c.Response().Header().Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
+	return config.Message{
-	c.Response().WriteHeader(http.StatusOK)
-	greet := config.Message{
 		Event:     config.EventAPIConnected,
 		Timestamp: time.Now(),
 	}
+}
+
+func (b *API) handleStream(c echo.Context) error {
+	c.Response().Header().Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
+	c.Response().WriteHeader(http.StatusOK)
+	greet := b.getGreeting()
 	if err := json.NewEncoder(c.Response()).Encode(greet); err != nil {
 		return err
 	}
 	c.Response().Flush()
 	for {
+		// TODO: this causes issues, messages should be broadcasted to all connected clients
 		msg := b.Messages.Dequeue()
 		if msg != nil {
 			if err := json.NewEncoder(c.Response()).Encode(msg); err != nil {
@@ -128,3 +177,31 @@ func (b *API) handleStream(c echo.Context) error {
 		time.Sleep(200 * time.Millisecond)
 	}
 }
+
+func (b *API) handleWebsocketMessage(message config.Message, s *melody.Session) {
+	message.Channel = "api"
+	message.Protocol = "api"
+	message.Account = b.Account
+	message.ID = ""
+	message.Timestamp = time.Now()
+
+	data, err := json.Marshal(message)
+	if err != nil {
+		b.Log.Errorf("failed to encode message for loopback '%v'", message)
+		return
+	}
+	_ = b.mrouter.BroadcastOthers(data, s)
+
+	b.Log.Debugf("Sending websocket message from %s on %s to gateway", message.Username, "api")
+	b.Remote <- message
+}
+
+func (b *API) handleWebsocket(c echo.Context) error {
+	err := b.mrouter.HandleRequest(c.Response(), c.Request())
+	if err != nil {
+		b.Log.Errorf("error in websocket handling  '%v'", err)
+		return err
+	}
+
+	return nil
+}

bridge/bridge.go

@@ -4,6 +4,7 @@ import (
 	"log"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/sirupsen/logrus"
@@ -74,6 +75,7 @@ func (b *Bridge) joinChannels(channels map[string]config.ChannelInfo, exists map
 	for ID, channel := range channels {
 		if !exists[ID] {
 			b.Log.Infof("%s: joining %s (ID: %s)", b.Account, channel.Name, ID)
+			time.Sleep(time.Duration(b.GetInt("JoinDelay")) * time.Millisecond)
 			err := b.JoinChannel(channel)
 			if err != nil {
 				return err
@@ -84,8 +86,16 @@ func (b *Bridge) joinChannels(channels map[string]config.ChannelInfo, exists map
 	return nil
 }
 
+func (b *Bridge) GetConfigKey(key string) string {
+	return b.Account + "." + key
+}
+
+func (b *Bridge) IsKeySet(key string) bool {
+	return b.Config.IsKeySet(b.GetConfigKey(key)) || b.Config.IsKeySet("general."+key)
+}
+
 func (b *Bridge) GetBool(key string) bool {
-	val, ok := b.Config.GetBool(b.Account + "." + key)
+	val, ok := b.Config.GetBool(b.GetConfigKey(key))
 	if !ok {
 		val, _ = b.Config.GetBool("general." + key)
 	}
@@ -93,7 +103,7 @@ func (b *Bridge) GetBool(key string) bool {
 }
 
 func (b *Bridge) GetInt(key string) int {
-	val, ok := b.Config.GetInt(b.Account + "." + key)
+	val, ok := b.Config.GetInt(b.GetConfigKey(key))
 	if !ok {
 		val, _ = b.Config.GetInt("general." + key)
 	}
@@ -101,7 +111,7 @@ func (b *Bridge) GetInt(key string) int {
 }
 
 func (b *Bridge) GetString(key string) string {
-	val, ok := b.Config.GetString(b.Account + "." + key)
+	val, ok := b.Config.GetString(b.GetConfigKey(key))
 	if !ok {
 		val, _ = b.Config.GetString("general." + key)
 	}
@@ -109,7 +119,7 @@ func (b *Bridge) GetString(key string) string {
 }
 
 func (b *Bridge) GetStringSlice(key string) []string {
-	val, ok := b.Config.GetStringSlice(b.Account + "." + key)
+	val, ok := b.Config.GetStringSlice(b.GetConfigKey(key))
 	if !ok {
 		val, _ = b.Config.GetStringSlice("general." + key)
 	}
@@ -117,7 +127,7 @@ func (b *Bridge) GetStringSlice(key string) []string {
 }
 
 func (b *Bridge) GetStringSlice2D(key string) [][]string {
-	val, ok := b.Config.GetStringSlice2D(b.Account + "." + key)
+	val, ok := b.Config.GetStringSlice2D(b.GetConfigKey(key))
 	if !ok {
 		val, _ = b.Config.GetStringSlice2D("general." + key)
 	}

bridge/config/config.go

@@ -3,6 +3,8 @@ package config
 import (
 	"bytes"
 	"io/ioutil"
+	"os"
+	"path/filepath"
 	"strings"
 	"sync"
 	"time"
@@ -24,8 +26,11 @@ const (
 	EventAPIConnected      = "api_connected"
 	EventUserTyping        = "user_typing"
 	EventGetChannelMembers = "get_channel_members"
+	EventNoticeIRC         = "notice_irc"
 )
 
+const ParentIDNotFound = "msg-parent-not-found"
+
 type Message struct {
 	Text      string    `json:"text"`
 	Channel   string    `json:"channel"`
@@ -42,6 +47,14 @@ type Message struct {
 	Extra     map[string][]interface{}
 }
 
+func (m Message) ParentNotFound() bool {
+	return m.ParentID == ParentIDNotFound
+}
+
+func (m Message) ParentValid() bool {
+	return m.ParentID != "" && !m.ParentNotFound()
+}
+
 type FileInfo struct {
 	Name    string
 	Data    *[]byte
@@ -76,24 +89,29 @@ type Protocol struct {
 	BindAddress            string // mattermost, slack // DEPRECATED
 	Buffer                 int    // api
 	Charset                string // irc
+	ClientID               string // msteams
 	ColorNicks             bool   // only irc for now
 	Debug                  bool   // general
 	DebugLevel             int    // only for irc now
 	DisableWebPagePreview  bool   // telegram
 	EditSuffix             string // mattermost, slack, discord, telegram, gitter
 	EditDisable            bool   // mattermost, slack, discord, telegram, gitter
+	HTMLDisable            bool   // matrix
 	IconURL                string // mattermost, slack
 	IgnoreFailureOnStart   bool   // general
 	IgnoreNicks            string // all protocols
 	IgnoreMessages         string // all protocols
 	Jid                    string // xmpp
+	JoinDelay              string // all protocols
 	Label                  string // all protocols
 	Login                  string // mattermost, matrix
+	LogFile                string // general
 	MediaDownloadBlackList []string
 	MediaDownloadPath      string // Basically MediaServerUpload, but instead of uploading it, just write it to a file on the same server.
 	MediaDownloadSize      int    // all protocols
 	MediaServerDownload    string
 	MediaServerUpload      string
+	MediaConvertTgs        string     // telegram
 	MediaConvertWebPToPNG  bool       // telegram
 	MessageDelay           int        // IRC, time in millisecond to wait between messages
 	MessageFormat          string     // telegram
@@ -110,7 +128,7 @@ type Protocol struct {
 	NicksPerRow            int        // mattermost, slack
 	NoHomeServerSuffix     bool       // matrix
 	NoSendJoinPart         bool       // all protocols
-	NoTLS                  bool       // mattermost
+	NoTLS                  bool       // mattermost, xmpp
 	Password               string     // IRC,mattermost,XMPP,matrix
 	PrefixMessagesWithNick bool       // mattemost, slack
 	PreserveThreading      bool       // slack
@@ -124,6 +142,7 @@ type Protocol struct {
 	RemoteNickFormat       string     // all protocols
 	RunCommands            []string   // IRC
 	Server                 string     // IRC,mattermost,XMPP,discord
+	SessionFile            string     // msteams,whatsapp
 	ShowJoinPart           bool       // all protocols
 	ShowTopicChange        bool       // slack
 	ShowUserTyping         bool       // slack
@@ -131,18 +150,22 @@ type Protocol struct {
 	SkipTLSVerify          bool       // IRC, mattermost
 	SkipVersionCheck       bool       // mattermost
 	StripNick              bool       // all protocols
+	StripMarkdown          bool       // irc
 	SyncTopic              bool       // slack
 	TengoModifyMessage     string     // general
 	Team                   string     // mattermost, keybase
+	TeamID                 string     // msteams
+	TenantID               string     // msteams
 	Token                  string     // gitter, slack, discord, api
 	Topic                  string     // zulip
 	URL                    string     // mattermost, slack // DEPRECATED
 	UseAPI                 bool       // mattermost, slack
+	UseLocalAvatar         []string   // discord
 	UseSASL                bool       // IRC
 	UseTLS                 bool       // IRC
 	UseDiscriminator       bool       // discord
 	UseFirstName           bool       // telegram
-	UseUserName            bool       // discord
+	UseUserName            bool       // discord, matrix
 	UseInsecureURL         bool       // telegram
 	VerboseJoinPart        bool       // IRC
 	WebhookBindAddress     string     // mattermost, slack
@@ -201,6 +224,7 @@ type BridgeValues struct {
 	WhatsApp           map[string]Protocol // TODO is this struct used? Search for "SlackLegacy" for example didn't return any results
 	Zulip              map[string]Protocol
 	Keybase            map[string]Protocol
+	Mumble             map[string]Protocol
 	General            Protocol
 	Tengo              Tengo
 	Gateway            []Gateway
@@ -210,6 +234,7 @@ type BridgeValues struct {
 type Config interface {
 	Viper() *viper.Viper
 	BridgeValues() *BridgeValues
+	IsKeySet(key string) bool
 	GetBool(key string) (bool, bool)
 	GetInt(key string) (int, bool)
 	GetString(key string) (string, bool)
@@ -235,7 +260,17 @@ func NewConfig(rootLogger *logrus.Logger, cfgfile string) Config {
 		logger.Fatalf("Failed to read configuration file: %#v", err)
 	}
 
-	mycfg := newConfigFromString(logger, input)
+	cfgtype := detectConfigType(cfgfile)
+	mycfg := newConfigFromString(logger, input, cfgtype)
+	if mycfg.cv.General.LogFile != "" {
+		logfile, err := os.OpenFile(mycfg.cv.General.LogFile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600)
+		if err == nil {
+			logger.Info("Opening log file ", mycfg.cv.General.LogFile)
+			rootLogger.Out = logfile
+		} else {
+			logger.Warn("Failed to open ", mycfg.cv.General.LogFile)
+		}
+	}
 	if mycfg.cv.General.MediaDownloadSize == 0 {
 		mycfg.cv.General.MediaDownloadSize = 1000000
 	}
@@ -246,14 +281,26 @@ func NewConfig(rootLogger *logrus.Logger, cfgfile string) Config {
 	return mycfg
 }
 
+// detectConfigType detects JSON and YAML formats, defaults to TOML.
+func detectConfigType(cfgfile string) string {
+	fileExt := filepath.Ext(cfgfile)
+	switch fileExt {
+	case ".json":
+		return "json"
+	case ".yaml", ".yml":
+		return "yaml"
+	}
+	return "toml"
+}
+
 // NewConfigFromString instantiates a new configuration based on the specified string.
 func NewConfigFromString(rootLogger *logrus.Logger, input []byte) Config {
 	logger := rootLogger.WithFields(logrus.Fields{"prefix": "config"})
-	return newConfigFromString(logger, input)
+	return newConfigFromString(logger, input, "toml")
 }
 
-func newConfigFromString(logger *logrus.Entry, input []byte) *config {
+func newConfigFromString(logger *logrus.Entry, input []byte, cfgtype string) *config {
-	viper.SetConfigType("toml")
+	viper.SetConfigType(cfgtype)
 	viper.SetEnvPrefix("matterbridge")
 	viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_", "-", "_"))
 	viper.AutomaticEnv()
@@ -281,6 +328,12 @@ func (c *config) Viper() *viper.Viper {
 	return c.v
 }
 
+func (c *config) IsKeySet(key string) bool {
+	c.RLock()
+	defer c.RUnlock()
+	return c.v.IsSet(key)
+}
+
 func (c *config) GetBool(key string) (bool, bool) {
 	c.RLock()
 	defer c.RUnlock()
@@ -340,6 +393,11 @@ type TestConfig struct {
 	Overrides map[string]interface{}
 }
 
+func (c *TestConfig) IsKeySet(key string) bool {
+	_, ok := c.Overrides[key]
+	return ok || c.Config.IsKeySet(key)
+}
+
 func (c *TestConfig) GetBool(key string) (bool, bool) {
 	val, ok := c.Overrides[key]
 	if ok {

bridge/discord/discord.go

@@ -2,15 +2,15 @@ package bdiscord
 
 import (
 	"bytes"
-	"errors"
 	"fmt"
 	"strings"
 	"sync"
 
 	"github.com/42wim/matterbridge/bridge"
 	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/42wim/matterbridge/bridge/discord/transmitter"
 	"github.com/42wim/matterbridge/bridge/helper"
-	"github.com/bwmarrin/discordgo"
+	"github.com/matterbridge/discordgo"
 )
 
 const MessageLength = 1950
@@ -20,11 +20,9 @@ type Bdiscord struct {
 
 	c *discordgo.Session
 
-	nick            string
+	nick    string
-	guildID         string
+	userID  string
-	webhookID       string
+	guildID string
-	webhookToken    string
-	canEditWebhooks bool
 
 	channelsMutex  sync.RWMutex
 	channels       []*discordgo.Channel
@@ -33,6 +31,10 @@ type Bdiscord struct {
 	membersMutex  sync.RWMutex
 	userMemberMap map[string]*discordgo.Member
 	nickMemberMap map[string]*discordgo.Member
+
+	// Webhook specific logic
+	useAutoWebhooks bool
+	transmitter     *transmitter.Transmitter
 }
 
 func New(cfg *bridge.Config) bridge.Bridger {
@@ -40,23 +42,18 @@ func New(cfg *bridge.Config) bridge.Bridger {
 	b.userMemberMap = make(map[string]*discordgo.Member)
 	b.nickMemberMap = make(map[string]*discordgo.Member)
 	b.channelInfoMap = make(map[string]*config.ChannelInfo)
-	if b.GetString("WebhookURL") != "" {
+
-		b.Log.Debug("Configuring Discord Incoming Webhook")
+	b.useAutoWebhooks = b.GetBool("AutoWebhooks")
-		b.webhookID, b.webhookToken = b.splitURL(b.GetString("WebhookURL"))
+	if b.useAutoWebhooks {
+		b.Log.Debug("Using automatic webhooks")
 	}
 	return b
 }
 
 func (b *Bdiscord) Connect() error {
 	var err error
-	var guildFound bool
 	token := b.GetString("Token")
 	b.Log.Info("Connecting")
-	if b.GetString("WebhookURL") == "" {
-		b.Log.Info("Connecting using token")
-	} else {
-		b.Log.Info("Connecting using webhookurl (for posting) and token")
-	}
 	if !strings.HasPrefix(b.GetString("Token"), "Bot ") {
 		token = "Bot " + b.GetString("Token")
 	}
@@ -92,55 +89,108 @@ func (b *Bdiscord) Connect() error {
 	}
 	serverName := strings.Replace(b.GetString("Server"), "ID:", "", -1)
 	b.nick = userinfo.Username
+	b.userID = userinfo.ID
+
+	// Try and find this account's guild, and populate channels
 	b.channelsMutex.Lock()
 	for _, guild := range guilds {
-		if guild.Name == serverName || guild.ID == serverName {
+		// Skip, if the server name does not match the visible name or the ID
-			b.channels, err = b.c.GuildChannels(guild.ID)
+		if guild.Name != serverName && guild.ID != serverName {
-			if err != nil {
+			continue
-				break
-			}
-			b.guildID = guild.ID
-			guildFound = true
 		}
+
+		// Complain about an ambiguous Server setting. Two Discord servers could have the same title!
+		// For IDs, practically this will never happen. It would only trigger if some server's name is also an ID.
+		if b.guildID != "" {
+			return fmt.Errorf("found multiple Discord servers with the same name %#v, expected to see only one", serverName)
+		}
+
+		// Getting this guild's channel could result in a permission error
+		b.channels, err = b.c.GuildChannels(guild.ID)
+		if err != nil {
+			return fmt.Errorf("could not get %#v's channels: %w", b.GetString("Server"), err)
+		}
+
+		b.guildID = guild.ID
 	}
 	b.channelsMutex.Unlock()
-	if !guildFound {
-		msg := fmt.Sprintf("Server \"%s\" not found", b.GetString("Server"))
-		err = errors.New(msg)
-		b.Log.Error(msg)
-		b.Log.Info("Possible values:")
-		for _, guild := range guilds {
-			b.Log.Infof("Server=\"%s\" # Server name", guild.Name)
-			b.Log.Infof("Server=\"%s\" # Server ID", guild.ID)
-		}
-	}
 
-	if err != nil {
+	// If we couldn't find a guild, we print extra debug information and return a nice error
+	if b.guildID == "" {
+		err = fmt.Errorf("could not find Discord server %#v", b.GetString("Server"))
+		b.Log.Error(err.Error())
+
+		// Print all of the possible server values
+		b.Log.Info("Possible server values:")
+		for _, guild := range guilds {
+			b.Log.Infof("\t- Server=%#v # by name", guild.Name)
+			b.Log.Infof("\t- Server=%#v # by ID", guild.ID)
+		}
+
+		// If there are no results, we should say that
+		if len(guilds) == 0 {
+			b.Log.Info("\t- (none found)")
+		}
+
 		return err
 	}
-	b.channelsMutex.RLock()
+
-	if b.GetString("WebhookURL") == "" {
+	// Legacy note: WebhookURL used to have an actual webhook URL that we would edit,
-		for _, channel := range b.channels {
+	// but we stopped doing that due to Discord making rate limits more aggressive.
-			b.Log.Debugf("found channel %#v", channel)
+	//
-		}
+	// Even older: the same WebhookURL used to be used by every channel, which is usually unexpected.
-	} else {
+	// This is no longer possible.
-		b.canEditWebhooks = true
+	if b.GetString("WebhookURL") != "" {
-		for _, channel := range b.channels {
+		message := "The global WebhookURL setting has been removed. "
-			b.Log.Debugf("found channel %#v; verifying PermissionManageWebhooks", channel)
+		message += "You can get similar \"webhook editing\" behaviour by replacing this line with `AutoWebhooks=true`. "
-			perms, permsErr := b.c.State.UserChannelPermissions(userinfo.ID, channel.ID)
+		message += "If you rely on the old-OLD (non-editing) behaviour, can move the WebhookURL to specific channel sections."
-			manageWebhooks := discordgo.PermissionManageWebhooks
+		b.Log.Errorln(message)
-			if permsErr != nil || perms&manageWebhooks != manageWebhooks {
+		return fmt.Errorf("use of removed WebhookURL setting")
-				b.Log.Warnf("Can't manage webhooks in channel \"%s\"", channel.Name)
+	}
-				b.canEditWebhooks = false
+
+	if b.GetInt("debuglevel") > 0 {
+		b.Log.Debug("enabling even more discord debug")
+		b.c.Debug = true
+	}
+
+	// Initialise webhook management
+	b.transmitter = transmitter.New(b.c, b.guildID, "matterbridge", b.useAutoWebhooks)
+	b.transmitter.Log = b.Log
+
+	var webhookChannelIDs []string
+	for _, channel := range b.Channels {
+		channelID := b.getChannelID(channel.Name) // note(qaisjp): this readlocks channelsMutex
+
+		// If a WebhookURL was not explicitly provided for this channel,
+		// there are two options: just a regular bot message (ugly) or this is should be webhook sent
+		if channel.Options.WebhookURL == "" {
+			// If it should be webhook sent, we should enforce this via the transmitter
+			if b.useAutoWebhooks {
+				webhookChannelIDs = append(webhookChannelIDs, channelID)
 			}
+			continue
 		}
-		if b.canEditWebhooks {
+
-			b.Log.Info("Can manage webhooks; will edit channel for global webhook on send")
+		whID, whToken, ok := b.splitURL(channel.Options.WebhookURL)
-		} else {
+		if !ok {
-			b.Log.Warn("Can't manage webhooks; won't edit channel for global webhook on send")
+			return fmt.Errorf("failed to parse WebhookURL %#v for channel %#v", channel.Options.WebhookURL, channel.ID)
+		}
+
+		b.transmitter.AddWebhook(channelID, &discordgo.Webhook{
+			ID:        whID,
+			Token:     whToken,
+			GuildID:   b.guildID,
+			ChannelID: channelID,
+		})
+	}
+
+	if b.useAutoWebhooks {
+		err = b.transmitter.RefreshGuildWebhooks(webhookChannelIDs)
+		if err != nil {
+			b.Log.WithError(err).Println("transmitter could not refresh guild webhooks")
+			return err
 		}
 	}
-	b.channelsMutex.RUnlock()
 
 	// Obtaining guild members and initializing nickname mapping.
 	b.membersMutex.Lock()
@@ -197,76 +247,23 @@ func (b *Bdiscord) Send(msg config.Message) (string, error) {
 		msg.Text = "_" + msg.Text + "_"
 	}
 
-	// use initial webhook configured for the entire Discord account
+	// Handle prefix hint for unthreaded messages.
-	isGlobalWebhook := true
+	if msg.ParentNotFound() {
-	wID := b.webhookID
+		msg.ParentID = ""
-	wToken := b.webhookToken
+		msg.Text = fmt.Sprintf("[thread]: %s", msg.Text)
-
-	// check if have a channel specific webhook
-	b.channelsMutex.RLock()
-	if ci, ok := b.channelInfoMap[msg.Channel+b.Account]; ok {
-		if ci.Options.WebhookURL != "" {
-			wID, wToken = b.splitURL(ci.Options.WebhookURL)
-			isGlobalWebhook = false
-		}
 	}
-	b.channelsMutex.RUnlock()
 
 	// Use webhook to send the message
-	if wID != "" && msg.Event != config.EventMsgDelete {
+	useWebhooks := b.shouldMessageUseWebhooks(&msg)
-		// skip events
+	if useWebhooks && msg.Event != config.EventMsgDelete && msg.ParentID == "" {
-		if msg.Event != "" && msg.Event != config.EventJoinLeave && msg.Event != config.EventTopicChange {
+		return b.handleEventWebhook(&msg, channelID)
-			return "", nil
-		}
-
-		// If we are editing a message, delete the old message
-		if msg.ID != "" {
-			b.Log.Debugf("Deleting edited webhook message")
-			err := b.c.ChannelMessageDelete(channelID, msg.ID)
-			if err != nil {
-				b.Log.Errorf("Could not delete edited webhook message: %s", err)
-			}
-		}
-
-		b.Log.Debugf("Broadcasting using Webhook")
-
-		// skip empty messages
-		if msg.Text == "" && (msg.Extra == nil || len(msg.Extra["file"]) == 0) {
-			b.Log.Debugf("Skipping empty message %#v", msg)
-			return "", nil
-		}
-
-		msg.Text = helper.ClipMessage(msg.Text, MessageLength)
-		msg.Text = b.replaceUserMentions(msg.Text)
-		// discord username must be [0..32] max
-		if len(msg.Username) > 32 {
-			msg.Username = msg.Username[0:32]
-		}
-		// if we have a global webhook for this Discord account, and permission
-		// to modify webhooks (previously verified), then set its channel to
-		// the message channel before using it
-		// TODO: this isn't necessary if the last message from this webhook was
-		// sent to the current channel
-		if isGlobalWebhook && b.canEditWebhooks {
-			b.Log.Debugf("Setting webhook channel to \"%s\"", msg.Channel)
-			_, err := b.c.WebhookEdit(wID, "", "", channelID)
-			if err != nil {
-				b.Log.Errorf("Could not set webhook channel: %s", err)
-				return "", err
-			}
-		}
-		b.Log.Debugf("Processing webhook sending for message %#v", msg)
-		msg, err := b.webhookSend(&msg, wID, wToken)
-		if err != nil {
-			b.Log.Errorf("Could not broadcast via webook for message %#v: %s", msg, err)
-			return "", err
-		}
-		if msg == nil {
-			return "", nil
-		}
-		return msg.ID, nil
 	}
 
+	return b.handleEventBotUser(&msg, channelID)
+}
+
+// handleEventDirect handles events via the bot user
+func (b *Bdiscord) handleEventBotUser(msg *config.Message, channelID string) (string, error) {
 	b.Log.Debugf("Broadcasting using token (API)")
 
 	// Delete message
@@ -280,7 +277,7 @@ func (b *Bdiscord) Send(msg config.Message) (string, error) {
 
 	// Upload a file if it exists
 	if msg.Extra != nil {
-		for _, rmsg := range helper.HandleExtra(&msg, b.General) {
+		for _, rmsg := range helper.HandleExtra(msg, b.General) {
 			rmsg.Text = helper.ClipMessage(rmsg.Text, MessageLength)
 			if _, err := b.c.ChannelMessageSend(channelID, rmsg.Username+rmsg.Text); err != nil {
 				b.Log.Errorf("Could not send message %#v: %s", rmsg, err)
@@ -288,7 +285,7 @@ func (b *Bdiscord) Send(msg config.Message) (string, error) {
 		}
 		// check if we have files to upload (from slack, telegram or mattermost)
 		if len(msg.Extra["file"]) > 0 {
-			return b.handleUploadFile(&msg, channelID)
+			return b.handleUploadFile(msg, channelID)
 		}
 	}
 
@@ -301,54 +298,27 @@ func (b *Bdiscord) Send(msg config.Message) (string, error) {
 		return msg.ID, err
 	}
 
+	m := discordgo.MessageSend{
+		Content: msg.Username + msg.Text,
+	}
+
+	if msg.ParentValid() {
+		m.Reference = &discordgo.MessageReference{
+			MessageID: msg.ParentID,
+			ChannelID: channelID,
+			GuildID:   b.guildID,
+		}
+	}
+
 	// Post normal message
-	res, err := b.c.ChannelMessageSend(channelID, msg.Username+msg.Text)
+	res, err := b.c.ChannelMessageSendComplex(channelID, &m)
 	if err != nil {
 		return "", err
 	}
+
 	return res.ID, nil
 }
 
-// useWebhook returns true if we have a webhook defined somewhere
-func (b *Bdiscord) useWebhook() bool {
-	if b.GetString("WebhookURL") != "" {
-		return true
-	}
-
-	b.channelsMutex.RLock()
-	defer b.channelsMutex.RUnlock()
-
-	for _, channel := range b.channelInfoMap {
-		if channel.Options.WebhookURL != "" {
-			return true
-		}
-	}
-	return false
-}
-
-// isWebhookID returns true if the specified id is used in a defined webhook
-func (b *Bdiscord) isWebhookID(id string) bool {
-	if b.GetString("WebhookURL") != "" {
-		wID, _ := b.splitURL(b.GetString("WebhookURL"))
-		if wID == id {
-			return true
-		}
-	}
-
-	b.channelsMutex.RLock()
-	defer b.channelsMutex.RUnlock()
-
-	for _, channel := range b.channelInfoMap {
-		if channel.Options.WebhookURL != "" {
-			wID, _ := b.splitURL(channel.Options.WebhookURL)
-			if wID == id {
-				return true
-			}
-		}
-	}
-	return false
-}
-
 // handleUploadFile handles native upload of files
 func (b *Bdiscord) handleUploadFile(msg *config.Message, channelID string) (string, error) {
 	var err error
@@ -370,62 +340,3 @@ func (b *Bdiscord) handleUploadFile(msg *config.Message, channelID string) (stri
 	}
 	return "", nil
 }
-
-// webhookSend send one or more message via webhook, taking care of file
-// uploads (from slack, telegram or mattermost).
-// Returns messageID and error.
-func (b *Bdiscord) webhookSend(msg *config.Message, webhookID, token string) (*discordgo.Message, error) {
-	var (
-		res *discordgo.Message
-		err error
-	)
-
-	// WebhookParams can have either `Content` or `File`.
-
-	// We can't send empty messages.
-	if msg.Text != "" {
-		res, err = b.c.WebhookExecute(
-			webhookID,
-			token,
-			true,
-			&discordgo.WebhookParams{
-				Content:   msg.Text,
-				Username:  msg.Username,
-				AvatarURL: msg.Avatar,
-			},
-		)
-		if err != nil {
-			b.Log.Errorf("Could not send text (%s) for message %#v: %s", msg.Text, msg, err)
-		}
-	}
-
-	if msg.Extra != nil {
-		for _, f := range msg.Extra["file"] {
-			fi := f.(config.FileInfo)
-			file := discordgo.File{
-				Name:        fi.Name,
-				ContentType: "",
-				Reader:      bytes.NewReader(*fi.Data),
-			}
-			content := ""
-			if msg.Text == "" {
-				content = fi.Comment
-			}
-			_, e2 := b.c.WebhookExecute(
-				webhookID,
-				token,
-				false,
-				&discordgo.WebhookParams{
-					Username:  msg.Username,
-					AvatarURL: msg.Avatar,
-					File:      &file,
-					Content:   content,
-				},
-			)
-			if e2 != nil {
-				b.Log.Errorf("Could not send file %#v for message %#v: %s", file, msg, e2)
-			}
-		}
-	}
-	return res, err
-}

bridge/discord/handlers.go

@@ -2,7 +2,7 @@ package bdiscord
 
 import (
 	"github.com/42wim/matterbridge/bridge/config"
-	"github.com/bwmarrin/discordgo"
+	"github.com/matterbridge/discordgo"
 )
 
 func (b *Bdiscord) messageDelete(s *discordgo.Session, m *discordgo.MessageDelete) { //nolint:unparam
@@ -36,6 +36,11 @@ func (b *Bdiscord) messageTyping(s *discordgo.Session, m *discordgo.TypingStart)
 		return
 	}
 
+	// Ignore our own typing messages
+	if m.UserID == b.userID {
+		return
+	}
+
 	rmsg := config.Message{Account: b.Account, Event: config.EventUserTyping}
 	rmsg.Channel = b.getChannelName(m.ChannelID)
 	b.Remote <- rmsg
@@ -64,7 +69,7 @@ func (b *Bdiscord) messageCreate(s *discordgo.Session, m *discordgo.MessageCreat
 		return
 	}
 	// if using webhooks, do not relay if it's ours
-	if b.useWebhook() && m.Author.Bot && b.isWebhookID(m.Author.ID) {
+	if m.Author.Bot && b.transmitter.HasWebhook(m.Author.ID) {
 		return
 	}
 
@@ -90,12 +95,12 @@ func (b *Bdiscord) messageCreate(s *discordgo.Session, m *discordgo.MessageCreat
 	// set channel name
 	rmsg.Channel = b.getChannelName(m.ChannelID)
 
-	// set username
+	fromWebhook := m.WebhookID != ""
-	if !b.GetBool("UseUserName") {
+	if !fromWebhook && !b.GetBool("UseUserName") {
 		rmsg.Username = b.getNick(m.Author, m.GuildID)
 	} else {
 		rmsg.Username = m.Author.Username
-		if b.GetBool("UseDiscriminator") {
+		if !fromWebhook && b.GetBool("UseDiscriminator") {
 			rmsg.Username += "#" + m.Author.Discriminator
 		}
 	}
@@ -103,7 +108,7 @@ func (b *Bdiscord) messageCreate(s *discordgo.Session, m *discordgo.MessageCreat
 	// if we have embedded content add it to text
 	if b.GetBool("ShowEmbeds") && m.Message.Embeds != nil {
 		for _, embed := range m.Message.Embeds {
-			rmsg.Text = rmsg.Text + "embed: " + embed.Title + " - " + embed.Description + " - " + embed.URL + "\n"
+			rmsg.Text += handleEmbed(embed)
 		}
 	}
 
@@ -119,6 +124,14 @@ func (b *Bdiscord) messageCreate(s *discordgo.Session, m *discordgo.MessageCreat
 		rmsg.Event = config.EventUserAction
 	}
 
+	// Replace emotes
+	rmsg.Text = replaceEmotes(rmsg.Text)
+
+	// Add our parent id if it exists, and if it's not referring to a message in another channel
+	if ref := m.MessageReference; ref != nil && ref.ChannelID == m.ChannelID {
+		rmsg.ParentID = ref.MessageID
+	}
+
 	b.Log.Debugf("<= Sending message from %s on %s to gateway", m.Author.Username, b.Account)
 	b.Log.Debugf("<= Message is %#v", rmsg)
 	b.Remote <- rmsg
@@ -192,3 +205,33 @@ func (b *Bdiscord) memberRemove(s *discordgo.Session, m *discordgo.GuildMemberRe
 	b.Log.Debugf("<= Message is %#v", rmsg)
 	b.Remote <- rmsg
 }
+
+func handleEmbed(embed *discordgo.MessageEmbed) string {
+	var t []string
+	var result string
+
+	t = append(t, embed.Title)
+	t = append(t, embed.Description)
+	t = append(t, embed.URL)
+
+	i := 0
+	for _, e := range t {
+		if e == "" {
+			continue
+		}
+
+		i++
+		if i == 1 {
+			result += " embed: " + e
+			continue
+		}
+
+		result += " - " + e
+	}
+
+	if result != "" {
+		result += "\n"
+	}
+
+	return result
+}

bridge/discord/handlers_test.go

@@ -0,0 +1,58 @@
+package bdiscord
+
+import (
+	"testing"
+
+	"github.com/matterbridge/discordgo"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestHandleEmbed(t *testing.T) {
+	testcases := map[string]struct {
+		embed  *discordgo.MessageEmbed
+		result string
+	}{
+		"allempty": {
+			embed:  &discordgo.MessageEmbed{},
+			result: "",
+		},
+		"one": {
+			embed: &discordgo.MessageEmbed{
+				Title: "blah",
+			},
+			result: " embed: blah\n",
+		},
+		"two": {
+			embed: &discordgo.MessageEmbed{
+				Title:       "blah",
+				Description: "blah2",
+			},
+			result: " embed: blah - blah2\n",
+		},
+		"three": {
+			embed: &discordgo.MessageEmbed{
+				Title:       "blah",
+				Description: "blah2",
+				URL:         "blah3",
+			},
+			result: " embed: blah - blah2 - blah3\n",
+		},
+		"twob": {
+			embed: &discordgo.MessageEmbed{
+				Description: "blah2",
+				URL:         "blah3",
+			},
+			result: " embed: blah2 - blah3\n",
+		},
+		"oneb": {
+			embed: &discordgo.MessageEmbed{
+				URL: "blah3",
+			},
+			result: " embed: blah3\n",
+		},
+	}
+
+	for name, tc := range testcases {
+		assert.Equalf(t, tc.result, handleEmbed(tc.embed), "Testcases %s", name)
+	}
+}

bridge/discord/helpers.go

@@ -6,7 +6,7 @@ import (
 	"strings"
 	"unicode"
 
-	"github.com/bwmarrin/discordgo"
+	"github.com/matterbridge/discordgo"
 )
 
 func (b *Bdiscord) getNick(user *discordgo.User, guildID string) string {
@@ -137,6 +137,7 @@ var (
 	// See https://discordapp.com/developers/docs/reference#message-formatting.
 	channelMentionRE = regexp.MustCompile("<#[0-9]+>")
 	userMentionRE    = regexp.MustCompile("@[^@\n]{1,32}")
+	emoteRE          = regexp.MustCompile(`<a?(:\w+:)\d+>`)
 )
 
 func (b *Bdiscord) replaceChannelMentions(text string) string {
@@ -182,15 +183,20 @@ func (b *Bdiscord) replaceUserMentions(text string) string {
 	return userMentionRE.ReplaceAllStringFunc(text, replaceUserMentionFunc)
 }
 
+func replaceEmotes(text string) string {
+	return emoteRE.ReplaceAllString(text, "$1")
+}
+
 func (b *Bdiscord) replaceAction(text string) (string, bool) {
-	if strings.HasPrefix(text, "_") && strings.HasSuffix(text, "_") {
+	length := len(text)
-		return text[1 : len(text)-1], true
+	if length > 1 && text[0] == '_' && text[length-1] == '_' {
+		return text[1 : length-1], true
 	}
 	return text, false
 }
 
 // splitURL splits a webhookURL and returns the ID and token.
-func (b *Bdiscord) splitURL(url string) (string, string) {
+func (b *Bdiscord) splitURL(url string) (string, string, bool) {
 	const (
 		expectedWebhookSplitCount = 7
 		webhookIdxID              = 5
@@ -198,9 +204,9 @@ func (b *Bdiscord) splitURL(url string) (string, string) {
 	)
 	webhookURLSplit := strings.Split(url, "/")
 	if len(webhookURLSplit) != expectedWebhookSplitCount {
-		b.Log.Fatalf("%s is no correct discord WebhookURL", url)
+		return "", "", false
 	}
-	return webhookURLSplit[webhookIdxID], webhookURLSplit[webhookIdxToken]
+	return webhookURLSplit[webhookIdxID], webhookURLSplit[webhookIdxToken], true
 }
 
 func enumerateUsernames(s string) []string {

bridge/discord/transmitter/transmitter.go

@@ -0,0 +1,257 @@
+// Package transmitter provides functionality for transmitting
+// arbitrary webhook messages to Discord.
+//
+// The package provides the following functionality:
+//
+// - Creating new webhooks, whenever necessary
+// - Loading webhooks that we have previously created
+// - Sending new messages
+// - Editing messages, via message ID
+// - Deleting messages, via message ID
+//
+// The package has been designed for matterbridge, but with other
+// Go bots in mind. The public API should be matterbridge-agnostic.
+package transmitter
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/matterbridge/discordgo"
+	log "github.com/sirupsen/logrus"
+)
+
+// A Transmitter represents a message manager for a single guild.
+type Transmitter struct {
+	session    *discordgo.Session
+	guild      string
+	title      string
+	autoCreate bool
+
+	// channelWebhooks maps from a channel ID to a webhook instance
+	channelWebhooks map[string]*discordgo.Webhook
+
+	mutex sync.RWMutex
+
+	Log *log.Entry
+}
+
+// ErrWebhookNotFound is returned when a valid webhook for this channel/message combination does not exist
+var ErrWebhookNotFound = errors.New("webhook for this channel and message does not exist")
+
+// ErrPermissionDenied is returned if the bot does not have permission to manage webhooks.
+//
+// Bots can be granted a guild-wide permission and channel-specific permissions to manage webhooks.
+// Despite potentially having guild-wide permission, channel specific overrides could deny a bot's permission to manage webhooks.
+var ErrPermissionDenied = errors.New("missing 'Manage Webhooks' permission")
+
+// New returns a new Transmitter given a Discord session, guild ID, and title.
+func New(session *discordgo.Session, guild string, title string, autoCreate bool) *Transmitter {
+	return &Transmitter{
+		session:    session,
+		guild:      guild,
+		title:      title,
+		autoCreate: autoCreate,
+
+		channelWebhooks: make(map[string]*discordgo.Webhook),
+
+		Log: log.NewEntry(nil),
+	}
+}
+
+// Send transmits a message to the given channel with the provided webhook data, and waits until Discord responds with message data.
+func (t *Transmitter) Send(channelID string, params *discordgo.WebhookParams) (*discordgo.Message, error) {
+	wh, err := t.getOrCreateWebhook(channelID)
+	if err != nil {
+		return nil, err
+	}
+
+	msg, err := t.session.WebhookExecute(wh.ID, wh.Token, true, params)
+	if err != nil {
+		return nil, fmt.Errorf("execute failed: %w", err)
+	}
+
+	return msg, nil
+}
+
+// Edit will edit a message in a channel, if possible.
+func (t *Transmitter) Edit(channelID string, messageID string, params *discordgo.WebhookParams) error {
+	wh := t.getWebhook(channelID)
+
+	if wh == nil {
+		return ErrWebhookNotFound
+	}
+
+	uri := discordgo.EndpointWebhookToken(wh.ID, wh.Token) + "/messages/" + messageID
+	_, err := t.session.RequestWithBucketID("PATCH", uri, params, discordgo.EndpointWebhookToken("", ""))
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// HasWebhook checks whether the transmitter is using a particular webhook.
+func (t *Transmitter) HasWebhook(id string) bool {
+	t.mutex.RLock()
+	defer t.mutex.RUnlock()
+
+	for _, wh := range t.channelWebhooks {
+		if wh.ID == id {
+			return true
+		}
+	}
+
+	return false
+}
+
+// AddWebhook allows you to register a channel's webhook with the transmitter.
+func (t *Transmitter) AddWebhook(channelID string, webhook *discordgo.Webhook) bool {
+	t.Log.Debugf("Manually added webhook %#v to channel %#v", webhook.ID, channelID)
+	t.mutex.Lock()
+	defer t.mutex.Unlock()
+
+	_, replaced := t.channelWebhooks[channelID]
+	t.channelWebhooks[channelID] = webhook
+	return replaced
+}
+
+// RefreshGuildWebhooks loads "relevant" webhooks into the transmitter, with careful permission handling.
+//
+// Notes:
+//
+// - A webhook is "relevant" if it was created by this bot -- the ApplicationID should match the bot's ID.
+// - The term "having permission" means having the "Manage Webhooks" permission. See ErrPermissionDenied for more information.
+// - This function is additive and will not unload previously loaded webhooks.
+// - A nil channelIDs slice is treated the same as an empty one.
+//
+// If the bot has guild-wide permission:
+//
+// 1. it will load any "relevant" webhooks from the entire guild
+// 2. the given slice is ignored
+//
+// If the bot does not have guild-wide permission:
+//
+// 1. it will load any "relevant" webhooks in each channel
+// 2. a single error will be returned if any error occurs (incl. if there is no permission for any of these channels)
+//
+// If any channel has more than one "relevant" webhook, it will randomly pick one.
+func (t *Transmitter) RefreshGuildWebhooks(channelIDs []string) error {
+	t.Log.Debugln("Refreshing guild webhooks")
+
+	botID, err := getDiscordUserID(t.session)
+	if err != nil {
+		return fmt.Errorf("could not get current user: %w", err)
+	}
+
+	// Get all existing webhooks
+	hooks, err := t.session.GuildWebhooks(t.guild)
+	if err != nil {
+		switch {
+		case isDiscordPermissionError(err):
+			// We fallback on manually fetching hooks from individual channels
+			// if we don't have the "Manage Webhooks" permission globally.
+			// We can only do this if we were provided channelIDs, though.
+			if len(channelIDs) == 0 {
+				return ErrPermissionDenied
+			}
+			t.Log.Debugln("Missing global 'Manage Webhooks' permission, falling back on per-channel permission")
+			return t.fetchChannelsHooks(channelIDs, botID)
+		default:
+			return fmt.Errorf("could not get webhooks: %w", err)
+		}
+	}
+
+	t.Log.Debugln("Refreshing guild webhooks using global permission")
+	t.assignHooksByAppID(hooks, botID, false)
+	return nil
+}
+
+// createWebhook creates a webhook for a specific channel.
+func (t *Transmitter) createWebhook(channel string) (*discordgo.Webhook, error) {
+	t.mutex.Lock()
+	defer t.mutex.Unlock()
+
+	wh, err := t.session.WebhookCreate(channel, t.title+time.Now().Format(" 3:04:05PM"), "")
+	if err != nil {
+		return nil, err
+	}
+
+	t.channelWebhooks[channel] = wh
+	return wh, nil
+}
+
+func (t *Transmitter) getWebhook(channel string) *discordgo.Webhook {
+	t.mutex.RLock()
+	defer t.mutex.RUnlock()
+
+	return t.channelWebhooks[channel]
+}
+
+func (t *Transmitter) getOrCreateWebhook(channelID string) (*discordgo.Webhook, error) {
+	// If we have a webhook for this channel, immediately return it
+	wh := t.getWebhook(channelID)
+	if wh != nil {
+		return wh, nil
+	}
+
+	// Early exit if we don't want to automatically create one
+	if !t.autoCreate {
+		return nil, ErrWebhookNotFound
+	}
+
+	t.Log.Infof("Creating a webhook for %s\n", channelID)
+	wh, err := t.createWebhook(channelID)
+	if err != nil {
+		return nil, fmt.Errorf("could not create webhook: %w", err)
+	}
+
+	return wh, nil
+}
+
+// fetchChannelsHooks fetches hooks for the given channelIDs and calls assignHooksByAppID for each channel's hooks
+func (t *Transmitter) fetchChannelsHooks(channelIDs []string, botID string) error {
+	// For each channel, search for relevant hooks
+	var failedHooks []string
+	for _, channelID := range channelIDs {
+		hooks, err := t.session.ChannelWebhooks(channelID)
+		if err != nil {
+			failedHooks = append(failedHooks, "\n- "+channelID+": "+err.Error())
+			continue
+		}
+		t.assignHooksByAppID(hooks, botID, true)
+	}
+
+	// Compose an error if any hooks failed
+	if len(failedHooks) > 0 {
+		return errors.New("failed to fetch hooks:" + strings.Join(failedHooks, ""))
+	}
+
+	return nil
+}
+
+func (t *Transmitter) assignHooksByAppID(hooks []*discordgo.Webhook, appID string, channelTargeted bool) {
+	logLine := "Picking up webhook"
+	if channelTargeted {
+		logLine += " (channel targeted)"
+	}
+
+	t.mutex.Lock()
+	defer t.mutex.Unlock()
+
+	for _, wh := range hooks {
+		if wh.ApplicationID != appID {
+			continue
+		}
+
+		t.channelWebhooks[wh.ChannelID] = wh
+		t.Log.WithFields(log.Fields{
+			"id":      wh.ID,
+			"name":    wh.Name,
+			"channel": wh.ChannelID,
+		}).Println(logLine)
+	}
+}

bridge/discord/transmitter/utils.go

@@ -0,0 +1,32 @@
+package transmitter
+
+import (
+	"github.com/matterbridge/discordgo"
+)
+
+// isDiscordPermissionError returns false for nil, and true if a Discord RESTError with code discordgo.ErrorCodeMissionPermissions
+func isDiscordPermissionError(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	restErr, ok := err.(*discordgo.RESTError)
+	if !ok {
+		return false
+	}
+
+	return restErr.Message != nil && restErr.Message.Code == discordgo.ErrCodeMissingPermissions
+}
+
+// getDiscordUserID gets own user ID from state, and fallback on API request
+func getDiscordUserID(session *discordgo.Session) (string, error) {
+	if user := session.State.User; user != nil {
+		return user.ID, nil
+	}
+
+	user, err := session.User("@me")
+	if err != nil {
+		return "", err
+	}
+	return user.ID, nil
+}

bridge/discord/webhook.go

@@ -0,0 +1,147 @@
+package bdiscord
+
+import (
+	"bytes"
+
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/42wim/matterbridge/bridge/helper"
+	"github.com/matterbridge/discordgo"
+)
+
+// shouldMessageUseWebhooks checks if have a channel specific webhook, if we're not using auto webhooks
+func (b *Bdiscord) shouldMessageUseWebhooks(msg *config.Message) bool {
+	if b.useAutoWebhooks {
+		return true
+	}
+
+	b.channelsMutex.RLock()
+	defer b.channelsMutex.RUnlock()
+	if ci, ok := b.channelInfoMap[msg.Channel+b.Account]; ok {
+		if ci.Options.WebhookURL != "" {
+			return true
+		}
+	}
+	return false
+}
+
+// maybeGetLocalAvatar checks if UseLocalAvatar contains the message's
+// account or protocol, and if so, returns the Discord avatar (if exists)
+func (b *Bdiscord) maybeGetLocalAvatar(msg *config.Message) string {
+	for _, val := range b.GetStringSlice("UseLocalAvatar") {
+		if msg.Protocol != val && msg.Account != val {
+			continue
+		}
+
+		member, err := b.getGuildMemberByNick(msg.Username)
+		if err != nil {
+			return ""
+		}
+
+		return member.User.AvatarURL("")
+	}
+	return ""
+}
+
+// webhookSend send one or more message via webhook, taking care of file
+// uploads (from slack, telegram or mattermost).
+// Returns messageID and error.
+func (b *Bdiscord) webhookSend(msg *config.Message, channelID string) (*discordgo.Message, error) {
+	var (
+		res *discordgo.Message
+		err error
+	)
+
+	// If avatar is unset, mutate the message to include the local avatar (but only if settings say we should do this)
+	if msg.Avatar == "" {
+		msg.Avatar = b.maybeGetLocalAvatar(msg)
+	}
+
+	// WebhookParams can have either `Content` or `File`.
+
+	// We can't send empty messages.
+	if msg.Text != "" {
+		res, err = b.transmitter.Send(
+			channelID,
+			&discordgo.WebhookParams{
+				Content:   msg.Text,
+				Username:  msg.Username,
+				AvatarURL: msg.Avatar,
+			},
+		)
+		if err != nil {
+			b.Log.Errorf("Could not send text (%s) for message %#v: %s", msg.Text, msg, err)
+		}
+	}
+
+	if msg.Extra != nil {
+		for _, f := range msg.Extra["file"] {
+			fi := f.(config.FileInfo)
+			file := discordgo.File{
+				Name:        fi.Name,
+				ContentType: "",
+				Reader:      bytes.NewReader(*fi.Data),
+			}
+			content := ""
+			if msg.Text == "" {
+				content = fi.Comment
+			}
+			_, e2 := b.transmitter.Send(
+				channelID,
+				&discordgo.WebhookParams{
+					Username:  msg.Username,
+					AvatarURL: msg.Avatar,
+					File:      &file,
+					Content:   content,
+				},
+			)
+			if e2 != nil {
+				b.Log.Errorf("Could not send file %#v for message %#v: %s", file, msg, e2)
+			}
+		}
+	}
+	return res, err
+}
+
+func (b *Bdiscord) handleEventWebhook(msg *config.Message, channelID string) (string, error) {
+	// skip events
+	if msg.Event != "" && msg.Event != config.EventUserAction && msg.Event != config.EventJoinLeave && msg.Event != config.EventTopicChange {
+		return "", nil
+	}
+
+	// skip empty messages
+	if msg.Text == "" && (msg.Extra == nil || len(msg.Extra["file"]) == 0) {
+		b.Log.Debugf("Skipping empty message %#v", msg)
+		return "", nil
+	}
+
+	msg.Text = helper.ClipMessage(msg.Text, MessageLength)
+	msg.Text = b.replaceUserMentions(msg.Text)
+	// discord username must be [0..32] max
+	if len(msg.Username) > 32 {
+		msg.Username = msg.Username[0:32]
+	}
+
+	if msg.ID != "" {
+		b.Log.Debugf("Editing webhook message")
+		err := b.transmitter.Edit(channelID, msg.ID, &discordgo.WebhookParams{
+			Content:  msg.Text,
+			Username: msg.Username,
+		})
+		if err == nil {
+			return msg.ID, nil
+		}
+		b.Log.Errorf("Could not edit webhook message: %s", err)
+	}
+
+	b.Log.Debugf("Processing webhook sending for message %#v", msg)
+	discordMsg, err := b.webhookSend(msg, channelID)
+	if err != nil {
+		b.Log.Errorf("Could not broadcast via webhook for message %#v: %s", msg, err)
+		return "", err
+	}
+	if discordMsg == nil {
+		return "", nil
+	}
+
+	return discordMsg.ID, nil
+}

bridge/helper/helper.go

@@ -5,7 +5,10 @@ import (
 	"fmt"
 	"image/png"
 	"io"
+	"io/ioutil"
 	"net/http"
+	"os"
+	"os/exec"
 	"regexp"
 	"strings"
 	"time"
@@ -180,7 +183,7 @@ func ClipMessage(text string, length int) string {
 
 // ParseMarkdown takes in an input string as markdown and parses it to html
 func ParseMarkdown(input string) string {
-	extensions := parser.HardLineBreak
+	extensions := parser.HardLineBreak | parser.NoIntraEmphasis | parser.FencedCode
 	markdownParser := parser.NewWithExtensions(extensions)
 	renderer := html.NewRenderer(html.RendererOptions{
 		Flags: 0,
@@ -192,7 +195,7 @@ func ParseMarkdown(input string) string {
 	return res
 }
 
-// ConvertWebPToPNG convert input data (which should be WebP format to PNG format)
+// ConvertWebPToPNG converts input data (which should be WebP format) to PNG format
 func ConvertWebPToPNG(data *[]byte) error {
 	r := bytes.NewReader(*data)
 	m, err := webp.Decode(r)
@@ -207,3 +210,49 @@ func ConvertWebPToPNG(data *[]byte) error {
 	*data = w.Bytes()
 	return nil
 }
+
+// CanConvertTgsToX Checks whether the external command necessary for ConvertTgsToX works.
+func CanConvertTgsToX() error {
+	// We depend on the fact that `lottie_convert.py --help` has exit status 0.
+	// Hyrum's Law predicted this, and Murphy's Law predicts that this will break eventually.
+	// However, there is no alternative like `lottie_convert.py --is-properly-installed`
+	cmd := exec.Command("lottie_convert.py", "--help")
+	return cmd.Run()
+}
+
+// ConvertTgsToWebP convert input data (which should be tgs format) to WebP format
+// This relies on an external command, which is ugly, but works.
+func ConvertTgsToX(data *[]byte, outputFormat string, logger *logrus.Entry) error {
+	// lottie can't handle input from a pipe, so write to a temporary file:
+	tmpFile, err := ioutil.TempFile(os.TempDir(), "matterbridge-lottie-*.tgs")
+	if err != nil {
+		return err
+	}
+	tmpFileName := tmpFile.Name()
+	defer func() {
+		if removeErr := os.Remove(tmpFileName); removeErr != nil {
+			logger.Errorf("Could not delete temporary file %s: %v", tmpFileName, removeErr)
+		}
+	}()
+
+	if _, writeErr := tmpFile.Write(*data); writeErr != nil {
+		return writeErr
+	}
+	// Must close before calling lottie to avoid data races:
+	if closeErr := tmpFile.Close(); closeErr != nil {
+		return closeErr
+	}
+
+	// Call lottie to transform:
+	cmd := exec.Command("lottie_convert.py", "--input-format", "lottie", "--output-format", outputFormat, tmpFileName, "/dev/stdout")
+	cmd.Stderr = nil
+	// NB: lottie writes progress into to stderr in all cases.
+	stdout, stderr := cmd.Output()
+	if stderr != nil {
+		// 'stderr' already contains some parts of Stderr, because it was set to 'nil'.
+		return stderr
+	}
+
+	*data = stdout
+	return nil
+}

bridge/irc/handlers.go

@@ -10,8 +10,8 @@ import (
 
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/42wim/matterbridge/bridge/helper"
-	"github.com/dfordsoft/golib/ic"
 	"github.com/lrstanley/girc"
+	"github.com/missdeer/golib/ic"
 	"github.com/paulrosania/go-charset/charset"
 	"github.com/saintfish/chardet"
 
@@ -54,12 +54,12 @@ func (b *Birc) handleFiles(msg *config.Message) bool {
 	for _, f := range msg.Extra["file"] {
 		fi := f.(config.FileInfo)
 		if fi.Comment != "" {
-			msg.Text += fi.Comment + ": "
+			msg.Text += fi.Comment + " : "
 		}
 		if fi.URL != "" {
 			msg.Text = fi.URL
 			if fi.Comment != "" {
-				msg.Text = fi.Comment + ": " + fi.URL
+				msg.Text = fi.Comment + " : " + fi.URL
 			}
 		}
 		b.Local <- config.Message{Text: msg.Text, Username: msg.Username, Channel: msg.Channel, Event: msg.Event}
@@ -67,6 +67,20 @@ func (b *Birc) handleFiles(msg *config.Message) bool {
 	return true
 }
 
+func (b *Birc) handleInvite(client *girc.Client, event girc.Event) {
+	if len(event.Params) != 2 {
+		return
+	}
+
+	channel := event.Params[1]
+
+	b.Log.Debugf("got invite for %s", channel)
+
+	if _, ok := b.channels[channel]; ok {
+		b.i.Cmd.Join(channel)
+	}
+}
+
 func (b *Birc) handleJoinPart(client *girc.Client, event girc.Event) {
 	if len(event.Params) == 0 {
 		b.Log.Debugf("handleJoinPart: empty Params? %#v", event)
@@ -109,14 +123,15 @@ func (b *Birc) handleNewConnection(client *girc.Client, event girc.Event) {
 	i := b.i
 	b.Nick = event.Params[0]
 
-	i.Handlers.Add("PRIVMSG", b.handlePrivMsg)
+	i.Handlers.AddBg("PRIVMSG", b.handlePrivMsg)
-	i.Handlers.Add("CTCP_ACTION", b.handlePrivMsg)
+	i.Handlers.AddBg("CTCP_ACTION", b.handlePrivMsg)
 	i.Handlers.Add(girc.RPL_TOPICWHOTIME, b.handleTopicWhoTime)
-	i.Handlers.Add(girc.NOTICE, b.handleNotice)
+	i.Handlers.AddBg(girc.NOTICE, b.handleNotice)
-	i.Handlers.Add("JOIN", b.handleJoinPart)
+	i.Handlers.AddBg("JOIN", b.handleJoinPart)
-	i.Handlers.Add("PART", b.handleJoinPart)
+	i.Handlers.AddBg("PART", b.handleJoinPart)
-	i.Handlers.Add("QUIT", b.handleJoinPart)
+	i.Handlers.AddBg("QUIT", b.handleJoinPart)
-	i.Handlers.Add("KICK", b.handleJoinPart)
+	i.Handlers.AddBg("KICK", b.handleJoinPart)
+	i.Handlers.Add("INVITE", b.handleInvite)
 }
 
 func (b *Birc) handleNickServ() {
@@ -170,7 +185,14 @@ func (b *Birc) handlePrivMsg(client *girc.Client, event girc.Event) {
 	if b.skipPrivMsg(event) {
 		return
 	}
-	rmsg := config.Message{Username: event.Source.Name, Channel: strings.ToLower(event.Params[0]), Account: b.Account, UserID: event.Source.Ident + "@" + event.Source.Host}
+
+	rmsg := config.Message{
+		Username: event.Source.Name,
+		Channel:  strings.ToLower(event.Params[0]),
+		Account:  b.Account,
+		UserID:   event.Source.Ident + "@" + event.Source.Host,
+	}
+
 	b.Log.Debugf("== Receiving PRIVMSG: %s %s %#v", event.Source.Name, event.Last(), event)
 
 	// set action event
@@ -178,6 +200,11 @@ func (b *Birc) handlePrivMsg(client *girc.Client, event girc.Event) {
 		rmsg.Event = config.EventUserAction
 	}
 
+	// set NOTICE event
+	if event.Command == "NOTICE" {
+		rmsg.Event = config.EventNoticeIRC
+	}
+
 	// strip action, we made an event if it was an action
 	rmsg.Text += event.StripAction()
 

bridge/irc/irc.go

@@ -4,6 +4,7 @@ import (
 	"crypto/tls"
 	"fmt"
 	"hash/crc32"
+	"io/ioutil"
 	"net"
 	"sort"
 	"strconv"
@@ -14,6 +15,7 @@ import (
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/42wim/matterbridge/bridge/helper"
 	"github.com/lrstanley/girc"
+	stripmd "github.com/writeas/go-strip-markdown"
 
 	// We need to import the 'data' package as an implicit dependency.
 	// See: https://godoc.org/github.com/paulrosania/go-charset/charset
@@ -28,6 +30,7 @@ type Birc struct {
 	Local                                     chan config.Message // local queue for flood control
 	FirstConnection, authDone                 bool
 	MessageDelay, MessageQueue, MessageLength int
+	channels                                  map[string]bool
 
 	*bridge.Config
 }
@@ -38,6 +41,8 @@ func New(cfg *bridge.Config) bridge.Bridger {
 	b.Nick = b.GetString("Nick")
 	b.names = make(map[string][]string)
 	b.connected = make(chan error)
+	b.channels = make(map[string]bool)
+
 	if b.GetInt("MessageDelay") == 0 {
 		b.MessageDelay = 1300
 	} else {
@@ -110,6 +115,7 @@ func (b *Birc) Disconnect() error {
 }
 
 func (b *Birc) JoinChannel(channel config.ChannelInfo) error {
+	b.channels[channel.Name] = true
 	// need to check if we have nickserv auth done before joining channels
 	for {
 		if b.authDone {
@@ -156,6 +162,10 @@ func (b *Birc) Send(msg config.Message) (string, error) {
 	}
 
 	var msgLines []string
+	if b.GetBool("StripMarkdown") {
+		msg.Text = stripmd.Strip(msg.Text)
+	}
+
 	if b.GetBool("MessageSplit") {
 		msgLines = helper.GetSubLines(msg.Text, b.MessageLength)
 	} else {
@@ -167,12 +177,8 @@ func (b *Birc) Send(msg config.Message) (string, error) {
 			return "", nil
 		}
 
-		b.Local <- config.Message{
+		msg.Text = msgLines[i]
-			Text:     msgLines[i],
+		b.Local <- msg
-			Username: msg.Username,
-			Channel:  msg.Channel,
-			Event:    msg.Event,
-		}
 	}
 	return "", nil
 }
@@ -199,22 +205,58 @@ func (b *Birc) doConnect() {
 	}
 }
 
+// Sanitize nicks for RELAYMSG: replace IRC characters with special meanings with "-"
+func sanitizeNick(nick string) string {
+	sanitize := func(r rune) rune {
+		if strings.ContainsRune("!+%@&#$:'\"?*,. ", r) {
+			return '-'
+		}
+		return r
+	}
+	return strings.Map(sanitize, nick)
+}
+
 func (b *Birc) doSend() {
 	rate := time.Millisecond * time.Duration(b.MessageDelay)
 	throttle := time.NewTicker(rate)
 	for msg := range b.Local {
 		<-throttle.C
 		username := msg.Username
-		if b.GetBool("Colornicks") {
+		// Optional support for the proposed RELAYMSG extension, described at
-			checksum := crc32.ChecksumIEEE([]byte(msg.Username))
+		// https://github.com/jlu5/ircv3-specifications/blob/master/extensions/relaymsg.md
-			colorCode := checksum%14 + 2 // quick fix - prevent white or black color codes
+		// nolint:nestif
-			username = fmt.Sprintf("\x03%02d%s\x0F", colorCode, msg.Username)
+		if (b.i.HasCapability("overdrivenetworks.com/relaymsg") || b.i.HasCapability("draft/relaymsg")) &&
-		}
+			b.GetBool("UseRelayMsg") {
-		if msg.Event == config.EventUserAction {
+			username = sanitizeNick(username)
-			b.i.Cmd.Action(msg.Channel, username+msg.Text)
+			text := msg.Text
+
+			// Work around girc chomping leading commas on single word messages?
+			if strings.HasPrefix(text, ":") && !strings.ContainsRune(text, ' ') {
+				text = ":" + text
+			}
+
+			if msg.Event == config.EventUserAction {
+				b.i.Cmd.SendRawf("RELAYMSG %s %s :\x01ACTION %s\x01", msg.Channel, username, text) //nolint:errcheck
+			} else {
+				b.Log.Debugf("Sending RELAYMSG to channel %s: nick=%s", msg.Channel, username)
+				b.i.Cmd.SendRawf("RELAYMSG %s %s :%s", msg.Channel, username, text) //nolint:errcheck
+			}
 		} else {
-			b.Log.Debugf("Sending to channel %s", msg.Channel)
+			if b.GetBool("Colornicks") {
-			b.i.Cmd.Message(msg.Channel, username+msg.Text)
+				checksum := crc32.ChecksumIEEE([]byte(msg.Username))
+				colorCode := checksum%14 + 2 // quick fix - prevent white or black color codes
+				username = fmt.Sprintf("\x03%02d%s\x0F", colorCode, msg.Username)
+			}
+			switch msg.Event {
+			case config.EventUserAction:
+				b.i.Cmd.Action(msg.Channel, username+msg.Text)
+			case config.EventNoticeIRC:
+				b.Log.Debugf("Sending notice to channel %s", msg.Channel)
+				b.i.Cmd.Notice(msg.Channel, username+msg.Text)
+			default:
+				b.Log.Debugf("Sending to channel %s", msg.Channel)
+				b.i.Cmd.Message(msg.Channel, username+msg.Text)
+			}
 		}
 	}
 }
@@ -239,6 +281,18 @@ func (b *Birc) getClient() (*girc.Client, error) {
 		user = user[1:]
 	}
 
+	debug := ioutil.Discard
+	if b.GetInt("DebugLevel") == 2 {
+		debug = b.Log.Writer()
+	}
+
+	pingDelay, err := time.ParseDuration(b.GetString("pingdelay"))
+	if err != nil || pingDelay == 0 {
+		pingDelay = time.Minute
+	}
+
+	b.Log.Debugf("setting pingdelay to %s", pingDelay)
+
 	i := girc.New(girc.Config{
 		Server:     server,
 		ServerPass: b.GetString("Password"),
@@ -248,7 +302,11 @@ func (b *Birc) getClient() (*girc.Client, error) {
 		Name:       b.GetString("Nick"),
 		SSL:        b.GetBool("UseTLS"),
 		TLSConfig:  &tls.Config{InsecureSkipVerify: b.GetBool("SkipTLSVerify"), ServerName: server}, //nolint:gosec
-		PingDelay:  time.Minute,
+		PingDelay:  pingDelay,
+		// skip gIRC internal rate limiting, since we have our own throttling
+		AllowFlood:    true,
+		Debug:         debug,
+		SupportedCaps: map[string][]string{"overdrivenetworks.com/relaymsg": nil, "draft/relaymsg": nil},
 	})
 	return i, nil
 }
@@ -274,7 +332,7 @@ func (b *Birc) skipPrivMsg(event girc.Event) bool {
 	b.Nick = b.i.GetNick()
 
 	// freenode doesn't send 001 as first reply
-	if event.Command == "NOTICE" {
+	if event.Command == "NOTICE" && len(event.Params) != 2 {
 		return true
 	}
 	// don't forward queries to the bot
@@ -285,6 +343,15 @@ func (b *Birc) skipPrivMsg(event girc.Event) bool {
 	if event.Source.Name == b.Nick {
 		return true
 	}
+	// don't forward messages we sent via RELAYMSG
+	if relayedNick, ok := event.Tags.Get("draft/relaymsg"); ok && relayedNick == b.Nick {
+		return true
+	}
+	// This is the old name of the cap sent in spoofed messages; I've kept this in
+	// for compatibility reasons
+	if relayedNick, ok := event.Tags.Get("relaymsg"); ok && relayedNick == b.Nick {
+		return true
+	}
 	return false
 }
 

bridge/keybase/handlers.go

@@ -4,7 +4,7 @@ import (
 	"strconv"
 
 	"github.com/42wim/matterbridge/bridge/config"
-	"github.com/keybase/go-keybase-chat-bot/kbchat"
+	"github.com/keybase/go-keybase-chat-bot/kbchat/types/chat1"
 )
 
 func (b *Bkeybase) handleKeybase() {
@@ -20,7 +20,7 @@ func (b *Bkeybase) handleKeybase() {
 				b.Log.Errorf("failed to read message: %s", err.Error())
 			}
 
-			if msg.Message.Content.Type != "text" {
+			if msg.Message.Content.TypeName != "text" {
 				continue
 			}
 
@@ -34,7 +34,7 @@ func (b *Bkeybase) handleKeybase() {
 	}()
 }
 
-func (b *Bkeybase) handleMessage(msg kbchat.Message) {
+func (b *Bkeybase) handleMessage(msg chat1.MsgSummary) {
 	b.Log.Debugf("== Receiving event: %#v", msg)
 	if msg.Channel.TopicName != b.channel || msg.Channel.Name != b.team {
 		return
@@ -45,10 +45,10 @@ func (b *Bkeybase) handleMessage(msg kbchat.Message) {
 		// TODO download avatar
 
 		// Create our message
-		rmsg := config.Message{Username: msg.Sender.Username, Text: msg.Content.Text.Body, UserID: msg.Sender.Uid, Channel: msg.Channel.TopicName, ID: strconv.Itoa(msg.MsgID), Account: b.Account}
+		rmsg := config.Message{Username: msg.Sender.Username, Text: msg.Content.Text.Body, UserID: string(msg.Sender.Uid), Channel: msg.Channel.TopicName, ID: strconv.Itoa(int(msg.Id)), Account: b.Account}
 
 		// Text must be a string
-		if msg.Content.Type != "text" {
+		if msg.Content.TypeName != "text" {
 			b.Log.Errorf("message is not text")
 			return
 		}

bridge/keybase/keybase.go

@@ -90,16 +90,17 @@ func (b *Bkeybase) Send(msg config.Message) (string, error) {
 				return "", err
 			}
 
-			_, _ = b.kbc.SendAttachmentByTeam(b.team, fpath, fcaption, &b.channel)
+			_, _ = b.kbc.SendAttachmentByTeam(b.team, &b.channel, fpath, fcaption)
 		}
 
 		return "", nil
 	}
 
 	// Send regular message
-	resp, err := b.kbc.SendMessageByTeamName(b.team, msg.Username+msg.Text, &b.channel)
+	text := msg.Username + msg.Text
+	resp, err := b.kbc.SendMessageByTeamName(b.team, &b.channel, text)
 	if err != nil {
 		return "", err
 	}
-	return strconv.Itoa(resp.Result.MsgID), err
+	return strconv.Itoa(int(*resp.Result.MessageID)), err
 }

bridge/matrix/helpers.go

@@ -0,0 +1,215 @@
+package bmatrix
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"html"
+	"strings"
+	"time"
+
+	matrix "github.com/matrix-org/gomatrix"
+)
+
+func newMatrixUsername(username string) *matrixUsername {
+	mUsername := new(matrixUsername)
+
+	// check if we have a </tag>. if we have, we don't escape HTML. #696
+	if htmlTag.MatchString(username) {
+		mUsername.formatted = username
+		// remove the HTML formatting for beautiful push messages #1188
+		mUsername.plain = htmlReplacementTag.ReplaceAllString(username, "")
+	} else {
+		mUsername.formatted = html.EscapeString(username)
+		mUsername.plain = username
+	}
+
+	return mUsername
+}
+
+// getRoomID retrieves a matching room ID from the channel name.
+func (b *Bmatrix) getRoomID(channel string) string {
+	b.RLock()
+	defer b.RUnlock()
+	for ID, name := range b.RoomMap {
+		if name == channel {
+			return ID
+		}
+	}
+
+	return ""
+}
+
+// interface2Struct marshals and immediately unmarshals an interface.
+// Useful for converting map[string]interface{} to a struct.
+func interface2Struct(in interface{}, out interface{}) error {
+	jsonObj, err := json.Marshal(in)
+	if err != nil {
+		return err //nolint:wrapcheck
+	}
+
+	return json.Unmarshal(jsonObj, out)
+}
+
+// getDisplayName retrieves the displayName for mxid, querying the homserver if the mxid is not in the cache.
+func (b *Bmatrix) getDisplayName(mxid string) string {
+	if b.GetBool("UseUserName") {
+		return mxid[1:]
+	}
+
+	b.RLock()
+	if val, present := b.NicknameMap[mxid]; present {
+		b.RUnlock()
+
+		return val.displayName
+	}
+	b.RUnlock()
+
+	displayName, err := b.mc.GetDisplayName(mxid)
+	var httpError *matrix.HTTPError
+	if errors.As(err, &httpError) {
+		b.Log.Warnf("Couldn't retrieve the display name for %s", mxid)
+	}
+
+	if err != nil {
+		return b.cacheDisplayName(mxid, mxid[1:])
+	}
+
+	return b.cacheDisplayName(mxid, displayName.DisplayName)
+}
+
+// cacheDisplayName stores the mapping between a mxid and a display name, to be reused later without performing a query to the homserver.
+// Note that old entries are cleaned when this function is called.
+func (b *Bmatrix) cacheDisplayName(mxid string, displayName string) string {
+	now := time.Now()
+
+	// scan to delete old entries, to stop memory usage from becoming too high with old entries.
+	// In addition, we also detect if another user have the same username, and if so, we append their mxids to their usernames to differentiate them.
+	toDelete := []string{}
+	conflict := false
+
+	b.Lock()
+	for mxid, v := range b.NicknameMap {
+		// to prevent username reuse across matrix servers - or even on the same server, append
+		// the mxid to the username when there is a conflict
+		if v.displayName == displayName {
+			conflict = true
+			// TODO: it would be nice to be able to rename previous messages from this user.
+			// The current behavior is that only users with clashing usernames and *that have spoken since the bridge last started* will get their mxids shown, and I don't know if that's the expected behavior.
+			v.displayName = fmt.Sprintf("%s (%s)", displayName, mxid)
+			b.NicknameMap[mxid] = v
+		}
+
+		if now.Sub(v.lastUpdated) > 10*time.Minute {
+			toDelete = append(toDelete, mxid)
+		}
+	}
+
+	if conflict {
+		displayName = fmt.Sprintf("%s (%s)", displayName, mxid)
+	}
+
+	for _, v := range toDelete {
+		delete(b.NicknameMap, v)
+	}
+
+	b.NicknameMap[mxid] = NicknameCacheEntry{
+		displayName: displayName,
+		lastUpdated: now,
+	}
+	b.Unlock()
+
+	return displayName
+}
+
+// handleError converts errors into httpError.
+//nolint:exhaustivestruct
+func handleError(err error) *httpError {
+	var mErr matrix.HTTPError
+	if !errors.As(err, &mErr) {
+		return &httpError{
+			Err: "not a HTTPError",
+		}
+	}
+
+	var httpErr httpError
+
+	if err := json.Unmarshal(mErr.Contents, &httpErr); err != nil {
+		return &httpError{
+			Err: "unmarshal failed",
+		}
+	}
+
+	return &httpErr
+}
+
+func (b *Bmatrix) containsAttachment(content map[string]interface{}) bool {
+	// Skip empty messages
+	if content["msgtype"] == nil {
+		return false
+	}
+
+	// Only allow image,video or file msgtypes
+	if !(content["msgtype"].(string) == "m.image" ||
+		content["msgtype"].(string) == "m.video" ||
+		content["msgtype"].(string) == "m.file") {
+		return false
+	}
+
+	return true
+}
+
+// getAvatarURL returns the avatar URL of the specified sender.
+func (b *Bmatrix) getAvatarURL(sender string) string {
+	urlPath := b.mc.BuildURL("profile", sender, "avatar_url")
+
+	s := struct {
+		AvatarURL string `json:"avatar_url"`
+	}{}
+
+	err := b.mc.MakeRequest("GET", urlPath, nil, &s)
+	if err != nil {
+		b.Log.Errorf("getAvatarURL failed: %s", err)
+
+		return ""
+	}
+
+	url := strings.ReplaceAll(s.AvatarURL, "mxc://", b.GetString("Server")+"/_matrix/media/r0/thumbnail/")
+	if url != "" {
+		url += "?width=37&height=37&method=crop"
+	}
+
+	return url
+}
+
+// handleRatelimit handles the ratelimit errors and return if we're ratelimited and the amount of time to sleep
+func (b *Bmatrix) handleRatelimit(err error) (time.Duration, bool) {
+	httpErr := handleError(err)
+	if httpErr.Errcode != "M_LIMIT_EXCEEDED" {
+		return 0, false
+	}
+
+	b.Log.Debugf("ratelimited: %s", httpErr.Err)
+	b.Log.Infof("getting ratelimited by matrix, sleeping approx %d seconds before retrying", httpErr.RetryAfterMs/1000)
+
+	return time.Duration(httpErr.RetryAfterMs) * time.Millisecond, true
+}
+
+// retry function will check if we're ratelimited and retries again when backoff time expired
+// returns original error if not 429 ratelimit
+func (b *Bmatrix) retry(f func() error) error {
+	b.rateMutex.Lock()
+	defer b.rateMutex.Unlock()
+
+	for {
+		if err := f(); err != nil {
+			if backoff, ok := b.handleRatelimit(err); ok {
+				time.Sleep(backoff)
+			} else {
+				return err
+			}
+		} else {
+			return nil
+		}
+	}
+}

bridge/matrix/matrix.go

@@ -3,31 +3,72 @@ package bmatrix
 import (
 	"bytes"
 	"fmt"
-	"html"
 	"mime"
 	"regexp"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/42wim/matterbridge/bridge"
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/42wim/matterbridge/bridge/helper"
-	matrix "github.com/matterbridge/gomatrix"
+	matrix "github.com/matrix-org/gomatrix"
 )
 
+var (
+	htmlTag            = regexp.MustCompile("</.*?>")
+	htmlReplacementTag = regexp.MustCompile("<[^>]*>")
+)
+
+type NicknameCacheEntry struct {
+	displayName string
+	lastUpdated time.Time
+}
+
 type Bmatrix struct {
-	mc      *matrix.Client
+	mc          *matrix.Client
-	UserID  string
+	UserID      string
-	RoomMap map[string]string
+	NicknameMap map[string]NicknameCacheEntry
+	RoomMap     map[string]string
+	rateMutex   sync.RWMutex
 	sync.RWMutex
-	htmlTag *regexp.Regexp
 	*bridge.Config
 }
 
+type httpError struct {
+	Errcode      string `json:"errcode"`
+	Err          string `json:"error"`
+	RetryAfterMs int    `json:"retry_after_ms"`
+}
+
+type matrixUsername struct {
+	plain     string
+	formatted string
+}
+
+// SubTextMessage represents the new content of the message in edit messages.
+type SubTextMessage struct {
+	MsgType string `json:"msgtype"`
+	Body    string `json:"body"`
+}
+
+// MessageRelation explains how the current message relates to a previous message.
+// Notably used for message edits.
+type MessageRelation struct {
+	EventID string `json:"event_id"`
+	Type    string `json:"rel_type"`
+}
+
+type EditedMessage struct {
+	NewContent SubTextMessage  `json:"m.new_content"`
+	RelatedTo  MessageRelation `json:"m.relates_to"`
+	matrix.TextMessage
+}
+
 func New(cfg *bridge.Config) bridge.Bridger {
 	b := &Bmatrix{Config: cfg}
-	b.htmlTag = regexp.MustCompile("</.*?>")
 	b.RoomMap = make(map[string]string)
+	b.NicknameMap = make(map[string]NicknameCacheEntry)
 	return b
 }
 
@@ -39,9 +80,10 @@ func (b *Bmatrix) Connect() error {
 		return err
 	}
 	resp, err := b.mc.Login(&matrix.ReqLogin{
-		Type:     "m.login.password",
+		Type:       "m.login.password",
-		User:     b.GetString("Login"),
+		User:       b.GetString("Login"),
-		Password: b.GetString("Password"),
+		Password:   b.GetString("Password"),
+		Identifier: matrix.NewUserIdentifier(b.GetString("Login")),
 	})
 	if err != nil {
 		return err
@@ -58,14 +100,18 @@ func (b *Bmatrix) Disconnect() error {
 }
 
 func (b *Bmatrix) JoinChannel(channel config.ChannelInfo) error {
-	resp, err := b.mc.JoinRoom(channel.Name, "", nil)
+	return b.retry(func() error {
-	if err != nil {
+		resp, err := b.mc.JoinRoom(channel.Name, "", nil)
-		return err
+		if err != nil {
-	}
+			return err
-	b.Lock()
+		}
-	b.RoomMap[resp.RoomID] = channel.Name
+
-	b.Unlock()
+		b.Lock()
-	return err
+		b.RoomMap[resp.RoomID] = channel.Name
+		b.Unlock()
+
+		return nil
+	})
 }
 
 func (b *Bmatrix) Send(msg config.Message) (string, error) {
@@ -74,17 +120,30 @@ func (b *Bmatrix) Send(msg config.Message) (string, error) {
 	channel := b.getRoomID(msg.Channel)
 	b.Log.Debugf("Channel %s maps to channel id %s", msg.Channel, channel)
 
+	username := newMatrixUsername(msg.Username)
+
 	// Make a action /me of the message
 	if msg.Event == config.EventUserAction {
 		m := matrix.TextMessage{
-			MsgType: "m.emote",
+			MsgType:       "m.emote",
-			Body:    msg.Username + msg.Text,
+			Body:          username.plain + msg.Text,
+			FormattedBody: username.formatted + msg.Text,
 		}
-		resp, err := b.mc.SendMessageEvent(channel, "m.room.message", m)
+
-		if err != nil {
+		msgID := ""
-			return "", err
+
-		}
+		err := b.retry(func() error {
-		return resp.EventID, err
+			resp, err := b.mc.SendMessageEvent(channel, "m.room.message", m)
+			if err != nil {
+				return err
+			}
+
+			msgID = resp.EventID
+
+			return err
+		})
+
+		return msgID, err
 	}
 
 	// Delete message
@@ -92,17 +151,34 @@ func (b *Bmatrix) Send(msg config.Message) (string, error) {
 		if msg.ID == "" {
 			return "", nil
 		}
-		resp, err := b.mc.RedactEvent(channel, msg.ID, &matrix.ReqRedact{})
+
-		if err != nil {
+		msgID := ""
-			return "", err
+
-		}
+		err := b.retry(func() error {
-		return resp.EventID, err
+			resp, err := b.mc.RedactEvent(channel, msg.ID, &matrix.ReqRedact{})
+			if err != nil {
+				return err
+			}
+
+			msgID = resp.EventID
+
+			return err
+		})
+
+		return msgID, err
 	}
 
 	// Upload a file if it exists
 	if msg.Extra != nil {
 		for _, rmsg := range helper.HandleExtra(&msg, b.General) {
-			if _, err := b.mc.SendText(channel, rmsg.Username+rmsg.Text); err != nil {
+			rmsg := rmsg
+
+			err := b.retry(func() error {
+				_, err := b.mc.SendText(channel, rmsg.Username+rmsg.Text)
+
+				return err
+			})
+			if err != nil {
 				b.Log.Errorf("sendText failed: %s", err)
 			}
 		}
@@ -113,45 +189,105 @@ func (b *Bmatrix) Send(msg config.Message) (string, error) {
 	}
 
 	// Edit message if we have an ID
-	// matrix has no editing support
+	if msg.ID != "" {
+		rmsg := EditedMessage{TextMessage: matrix.TextMessage{
+			Body:    username.plain + msg.Text,
+			MsgType: "m.text",
+		}}
+		if b.GetBool("HTMLDisable") {
+			rmsg.TextMessage.FormattedBody = username.formatted + "* " + msg.Text
+		} else {
+			rmsg.Format = "org.matrix.custom.html"
+			rmsg.TextMessage.FormattedBody = username.formatted + "* " + helper.ParseMarkdown(msg.Text)
+		}
+		rmsg.NewContent = SubTextMessage{
+			Body:    rmsg.TextMessage.Body,
+			MsgType: "m.text",
+		}
+		rmsg.RelatedTo = MessageRelation{
+			EventID: msg.ID,
+			Type:    "m.replace",
+		}
 
-	// Use notices to send join/leave events
+		err := b.retry(func() error {
-	if msg.Event == config.EventJoinLeave {
+			_, err := b.mc.SendMessageEvent(channel, "m.room.message", rmsg)
-		resp, err := b.mc.SendNotice(channel, msg.Username+msg.Text)
+
+			return err
+		})
 		if err != nil {
 			return "", err
 		}
+
+		return msg.ID, nil
+	}
+
+	// Use notices to send join/leave events
+	if msg.Event == config.EventJoinLeave {
+		m := matrix.TextMessage{
+			MsgType:       "m.notice",
+			Body:          username.plain + msg.Text,
+			FormattedBody: username.formatted + msg.Text,
+		}
+
+		var (
+			resp *matrix.RespSendEvent
+			err  error
+		)
+
+		err = b.retry(func() error {
+			resp, err = b.mc.SendMessageEvent(channel, "m.room.message", m)
+
+			return err
+		})
+		if err != nil {
+			return "", err
+		}
+
 		return resp.EventID, err
 	}
 
-	username := html.EscapeString(msg.Username)
+	if b.GetBool("HTMLDisable") {
-	// check if we have a </tag>. if we have, we don't escape HTML. #696
+		var (
-	if b.htmlTag.MatchString(msg.Username) {
+			resp *matrix.RespSendEvent
-		username = msg.Username
+			err  error
+		)
+
+		err = b.retry(func() error {
+			resp, err = b.mc.SendText(channel, username.plain+msg.Text)
+
+			return err
+		})
+		if err != nil {
+			return "", err
+		}
+
+		return resp.EventID, err
 	}
+
 	// Post normal message with HTML support (eg riot.im)
-	resp, err := b.mc.SendHTML(channel, msg.Username+msg.Text, username+helper.ParseMarkdown(msg.Text))
+	var (
+		resp *matrix.RespSendEvent
+		err  error
+	)
+
+	err = b.retry(func() error {
+		resp, err = b.mc.SendFormattedText(channel, username.plain+msg.Text,
+			username.formatted+helper.ParseMarkdown(msg.Text))
+
+		return err
+	})
 	if err != nil {
 		return "", err
 	}
-	return resp.EventID, err
-}
 
-func (b *Bmatrix) getRoomID(channel string) string {
+	return resp.EventID, err
-	b.RLock()
-	defer b.RUnlock()
-	for ID, name := range b.RoomMap {
-		if name == channel {
-			return ID
-		}
-	}
-	return ""
 }
 
 func (b *Bmatrix) handlematrix() {
 	syncer := b.mc.Syncer.(*matrix.DefaultSyncer)
 	syncer.OnEventType("m.room.redaction", b.handleEvent)
 	syncer.OnEventType("m.room.message", b.handleEvent)
+	syncer.OnEventType("m.room.member", b.handleMemberChange)
 	go func() {
 		for {
 			if err := b.mc.Sync(); err != nil {
@@ -161,6 +297,45 @@ func (b *Bmatrix) handlematrix() {
 	}()
 }
 
+func (b *Bmatrix) handleEdit(ev *matrix.Event, rmsg config.Message) bool {
+	relationInterface, present := ev.Content["m.relates_to"]
+	newContentInterface, present2 := ev.Content["m.new_content"]
+	if !(present && present2) {
+		return false
+	}
+
+	var relation MessageRelation
+	if err := interface2Struct(relationInterface, &relation); err != nil {
+		b.Log.Warnf("Couldn't parse 'm.relates_to' object with value %#v", relationInterface)
+		return false
+	}
+
+	var newContent SubTextMessage
+	if err := interface2Struct(newContentInterface, &newContent); err != nil {
+		b.Log.Warnf("Couldn't parse 'm.new_content' object with value %#v", newContentInterface)
+		return false
+	}
+
+	if relation.Type != "m.replace" {
+		return false
+	}
+
+	rmsg.ID = relation.EventID
+	rmsg.Text = newContent.Body
+	b.Remote <- rmsg
+
+	return true
+}
+
+func (b *Bmatrix) handleMemberChange(ev *matrix.Event) {
+	// Update the displayname on join messages, according to https://matrix.org/docs/spec/client_server/r0.6.1#events-on-change-of-profile-information
+	if ev.Content["membership"] == "join" {
+		if dn, ok := ev.Content["displayname"].(string); ok {
+			b.cacheDisplayName(ev.Sender, dn)
+		}
+	}
+}
+
 func (b *Bmatrix) handleEvent(ev *matrix.Event) {
 	b.Log.Debugf("== Receiving event: %#v", ev)
 	if ev.Sender != b.UserID {
@@ -172,10 +347,15 @@ func (b *Bmatrix) handleEvent(ev *matrix.Event) {
 			return
 		}
 
-		// TODO download avatar
-
 		// Create our message
-		rmsg := config.Message{Username: ev.Sender[1:], Channel: channel, Account: b.Account, UserID: ev.Sender, ID: ev.ID}
+		rmsg := config.Message{
+			Username: b.getDisplayName(ev.Sender),
+			Channel:  channel,
+			Account:  b.Account,
+			UserID:   ev.Sender,
+			ID:       ev.ID,
+			Avatar:   b.getAvatarURL(ev.Sender),
+		}
 
 		// Text must be a string
 		if rmsg.Text, ok = ev.Content["body"].(string); !ok {
@@ -204,6 +384,11 @@ func (b *Bmatrix) handleEvent(ev *matrix.Event) {
 			rmsg.Event = config.EventUserAction
 		}
 
+		// Is it an edit?
+		if b.handleEdit(ev, rmsg) {
+			return
+		}
+
 		// Do we have attachments
 		if b.containsAttachment(ev.Content) {
 			err := b.handleDownloadFile(&rmsg, ev.Content)
@@ -214,6 +399,11 @@ func (b *Bmatrix) handleEvent(ev *matrix.Event) {
 
 		b.Log.Debugf("<= Sending message from %s on %s to gateway", ev.Sender, b.Account)
 		b.Remote <- rmsg
+
+		// not crucial, so no ratelimit check here
+		if err := b.mc.MarkRead(ev.RoomID, ev.ID); err != nil {
+			b.Log.Errorf("couldn't mark message as read %s", err.Error())
+		}
 	}
 }
 
@@ -288,27 +478,30 @@ func (b *Bmatrix) handleUploadFiles(msg *config.Message, channel string) (string
 
 // handleUploadFile handles native upload of a file.
 func (b *Bmatrix) handleUploadFile(msg *config.Message, channel string, fi *config.FileInfo) {
+	username := newMatrixUsername(msg.Username)
 	content := bytes.NewReader(*fi.Data)
 	sp := strings.Split(fi.Name, ".")
 	mtype := mime.TypeByExtension("." + sp[len(sp)-1])
-	if !(strings.Contains(mtype, "image") || strings.Contains(mtype, "video") ||
+	// image and video uploads send no username, we have to do this ourself here #715
-		strings.Contains(mtype, "application") || strings.Contains(mtype, "audio")) {
+	err := b.retry(func() error {
-		return
+		_, err := b.mc.SendFormattedText(channel, username.plain+fi.Comment, username.formatted+fi.Comment)
-	}
+
-	if fi.Comment != "" {
+		return err
-		_, err := b.mc.SendText(channel, msg.Username+fi.Comment)
+	})
-		if err != nil {
+	if err != nil {
-			b.Log.Errorf("file comment failed: %#v", err)
+		b.Log.Errorf("file comment failed: %#v", err)
-		}
-	} else {
-		// image and video uploads send no username, we have to do this ourself here #715
-		_, err := b.mc.SendText(channel, msg.Username)
-		if err != nil {
-			b.Log.Errorf("file comment failed: %#v", err)
-		}
 	}
+
 	b.Log.Debugf("uploading file: %s %s", fi.Name, mtype)
-	res, err := b.mc.UploadToContentRepo(content, mtype, int64(len(*fi.Data)))
+
+	var res *matrix.RespMediaUpload
+
+	err = b.retry(func() error {
+		res, err = b.mc.UploadToContentRepo(content, mtype, int64(len(*fi.Data)))
+
+		return err
+	})
+
 	if err != nil {
 		b.Log.Errorf("file upload failed: %#v", err)
 		return
@@ -317,44 +510,60 @@ func (b *Bmatrix) handleUploadFile(msg *config.Message, channel string, fi *conf
 	switch {
 	case strings.Contains(mtype, "video"):
 		b.Log.Debugf("sendVideo %s", res.ContentURI)
-		_, err = b.mc.SendVideo(channel, fi.Name, res.ContentURI)
+		err = b.retry(func() error {
+			_, err = b.mc.SendVideo(channel, fi.Name, res.ContentURI)
+
+			return err
+		})
 		if err != nil {
 			b.Log.Errorf("sendVideo failed: %#v", err)
 		}
 	case strings.Contains(mtype, "image"):
 		b.Log.Debugf("sendImage %s", res.ContentURI)
-		_, err = b.mc.SendImage(channel, fi.Name, res.ContentURI)
+		err = b.retry(func() error {
+			_, err = b.mc.SendImage(channel, fi.Name, res.ContentURI)
+
+			return err
+		})
 		if err != nil {
 			b.Log.Errorf("sendImage failed: %#v", err)
 		}
-	case strings.Contains(mtype, "application"):
-		b.Log.Debugf("sendFile %s", res.ContentURI)
-		_, err = b.mc.SendFile(channel, fi.Name, res.ContentURI, mtype, uint(len(*fi.Data)))
-		if err != nil {
-			b.Log.Errorf("sendFile failed: %#v", err)
-		}
 	case strings.Contains(mtype, "audio"):
 		b.Log.Debugf("sendAudio %s", res.ContentURI)
-		_, err = b.mc.SendAudio(channel, fi.Name, res.ContentURI, mtype, uint(len(*fi.Data)))
+		err = b.retry(func() error {
+			_, err = b.mc.SendMessageEvent(channel, "m.room.message", matrix.AudioMessage{
+				MsgType: "m.audio",
+				Body:    fi.Name,
+				URL:     res.ContentURI,
+				Info: matrix.AudioInfo{
+					Mimetype: mtype,
+					Size:     uint(len(*fi.Data)),
+				},
+			})
+
+			return err
+		})
 		if err != nil {
 			b.Log.Errorf("sendAudio failed: %#v", err)
 		}
+	default:
+		b.Log.Debugf("sendFile %s", res.ContentURI)
+		err = b.retry(func() error {
+			_, err = b.mc.SendMessageEvent(channel, "m.room.message", matrix.FileMessage{
+				MsgType: "m.file",
+				Body:    fi.Name,
+				URL:     res.ContentURI,
+				Info: matrix.FileInfo{
+					Mimetype: mtype,
+					Size:     uint(len(*fi.Data)),
+				},
+			})
+
+			return err
+		})
+		if err != nil {
+			b.Log.Errorf("sendFile failed: %#v", err)
+		}
 	}
 	b.Log.Debugf("result: %#v", res)
 }
-
-// skipMessages returns true if this message should not be handled
-func (b *Bmatrix) containsAttachment(content map[string]interface{}) bool {
-	// Skip empty messages
-	if content["msgtype"] == nil {
-		return false
-	}
-
-	// Only allow image,video or file msgtypes
-	if !(content["msgtype"].(string) == "m.image" ||
-		content["msgtype"].(string) == "m.video" ||
-		content["msgtype"].(string) == "m.file") {
-		return false
-	}
-	return true
-}

bridge/matrix/matrix_test.go

@@ -0,0 +1,28 @@
+package bmatrix
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestPlainUsername(t *testing.T) {
+	uut := newMatrixUsername("MyUser")
+
+	assert.Equal(t, "MyUser", uut.formatted)
+	assert.Equal(t, "MyUser", uut.plain)
+}
+
+func TestHTMLUsername(t *testing.T) {
+	uut := newMatrixUsername("<b>MyUser</b>")
+
+	assert.Equal(t, "<b>MyUser</b>", uut.formatted)
+	assert.Equal(t, "MyUser", uut.plain)
+}
+
+func TestFancyUsername(t *testing.T) {
+	uut := newMatrixUsername("<MyUser>")
+
+	assert.Equal(t, "&lt;MyUser&gt;", uut.formatted)
+	assert.Equal(t, "<MyUser>", uut.plain)
+}

bridge/mattermost/handlers.go

@@ -4,7 +4,7 @@ import (
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/42wim/matterbridge/bridge/helper"
 	"github.com/42wim/matterbridge/matterclient"
-	"github.com/mattermost/mattermost-server/model"
+	"github.com/mattermost/mattermost-server/v5/model"
 )
 
 // handleDownloadAvatar downloads the avatar of userid from channel
@@ -108,7 +108,7 @@ func (b *Bmattermost) handleMatterClient(messages chan *config.Message) {
 			Channel:  message.Channel,
 			Text:     message.Text,
 			ID:       message.Post.Id,
-			ParentID: message.Post.ParentId,
+			ParentID: message.Post.RootId, // ParentID is obsolete with mattermost
 			Extra:    make(map[string][]interface{}),
 		}
 

bridge/mattermost/helpers.go

@@ -7,7 +7,7 @@ import (
 	"github.com/42wim/matterbridge/bridge/helper"
 	"github.com/42wim/matterbridge/matterclient"
 	"github.com/42wim/matterbridge/matterhook"
-	"github.com/mattermost/mattermost-server/model"
+	"github.com/mattermost/mattermost-server/v5/model"
 )
 
 func (b *Bmattermost) doConnectWebhookBind() error {

bridge/mattermost/mattermost.go

@@ -122,11 +122,20 @@ func (b *Bmattermost) Send(msg config.Message) (string, error) {
 	}
 
 	// Handle prefix hint for unthreaded messages.
-	if msg.ParentID == "msg-parent-not-found" {
+	if msg.ParentNotFound() {
 		msg.ParentID = ""
 		msg.Text = fmt.Sprintf("[thread]: %s", msg.Text)
 	}
 
+	// we only can reply to the root of the thread, not to a specific ID (like discord for example does)
+	if msg.ParentID != "" {
+		post, res := b.mc.Client.GetPost(msg.ParentID, "")
+		if res.Error != nil {
+			b.Log.Errorf("getting post %s failed: %s", msg.ParentID, res.Error.DetailedError)
+		}
+		msg.ParentID = post.RootId
+	}
+
 	// Upload a file if it exists
 	if msg.Extra != nil {
 		for _, rmsg := range helper.HandleExtra(&msg, b.General) {

bridge/msteams/handler.go

@@ -0,0 +1,101 @@
+package bmsteams
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"strings"
+
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/42wim/matterbridge/bridge/helper"
+
+	msgraph "github.com/yaegashi/msgraph.go/beta"
+)
+
+func (b *Bmsteams) findFile(weburl string) (string, error) {
+	itemRB, err := b.gc.GetDriveItemByURL(b.ctx, weburl)
+	if err != nil {
+		return "", err
+	}
+	itemRB.Workbook().Worksheets()
+	b.gc.Workbooks()
+	item, err := itemRB.Request().Get(b.ctx)
+	if err != nil {
+		return "", err
+	}
+	if url, ok := item.GetAdditionalData("@microsoft.graph.downloadUrl"); ok {
+		return url.(string), nil
+	}
+	return "", nil
+}
+
+// handleDownloadFile handles file download
+func (b *Bmsteams) handleDownloadFile(rmsg *config.Message, filename, weburl string) error {
+	realURL, err := b.findFile(weburl)
+	if err != nil {
+		return err
+	}
+	// Actually download the file.
+	data, err := helper.DownloadFile(realURL)
+	if err != nil {
+		return fmt.Errorf("download %s failed %#v", weburl, err)
+	}
+
+	// If a comment is attached to the file(s) it is in the 'Text' field of the teams messge event
+	// and should be added as comment to only one of the files. We reset the 'Text' field to ensure
+	// that the comment is not duplicated.
+	comment := rmsg.Text
+	rmsg.Text = ""
+	helper.HandleDownloadData(b.Log, rmsg, filename, comment, weburl, data, b.General)
+	return nil
+}
+
+func (b *Bmsteams) handleAttachments(rmsg *config.Message, msg msgraph.ChatMessage) {
+	for _, a := range msg.Attachments {
+		//remove the attachment tags from the text
+		rmsg.Text = attachRE.ReplaceAllString(rmsg.Text, "")
+
+		//handle a code snippet (code block)
+		if *a.ContentType == "application/vnd.microsoft.card.codesnippet" {
+			b.handleCodeSnippet(rmsg, a)
+			continue
+		}
+
+		//handle the download
+		err := b.handleDownloadFile(rmsg, *a.Name, *a.ContentURL)
+		if err != nil {
+			b.Log.Errorf("download of %s failed: %s", *a.Name, err)
+		}
+	}
+}
+
+type AttachContent struct {
+	Language       string `json:"language"`
+	CodeSnippetURL string `json:"codeSnippetUrl"`
+}
+
+func (b *Bmsteams) handleCodeSnippet(rmsg *config.Message, attach msgraph.ChatMessageAttachment) {
+	var content AttachContent
+	err := json.Unmarshal([]byte(*attach.Content), &content)
+	if err != nil {
+		b.Log.Errorf("unmarshal codesnippet failed: %s", err)
+		return
+	}
+	s := strings.Split(content.CodeSnippetURL, "/")
+	if len(s) != 13 {
+		b.Log.Errorf("codesnippetUrl has unexpected size: %s", content.CodeSnippetURL)
+		return
+	}
+	resp, err := b.gc.Teams().Request().Client().Get(content.CodeSnippetURL)
+	if err != nil {
+		b.Log.Errorf("retrieving snippet content failed:%s", err)
+		return
+	}
+	defer resp.Body.Close()
+	res, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		b.Log.Errorf("reading snippet data failed: %s", err)
+		return
+	}
+	rmsg.Text = rmsg.Text + "\n```" + content.Language + "\n" + string(res) + "\n```\n"
+}

bridge/msteams/msteams.go

@@ -0,0 +1,227 @@
+package bmsteams
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/42wim/matterbridge/bridge"
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/davecgh/go-spew/spew"
+
+	"github.com/mattn/godown"
+	msgraph "github.com/yaegashi/msgraph.go/beta"
+	"github.com/yaegashi/msgraph.go/msauth"
+
+	"golang.org/x/oauth2"
+)
+
+var defaultScopes = []string{"openid", "profile", "offline_access", "Group.Read.All", "Group.ReadWrite.All"}
+var attachRE = regexp.MustCompile(`<attachment id=.*?attachment>`)
+
+type Bmsteams struct {
+	gc    *msgraph.GraphServiceRequestBuilder
+	ctx   context.Context
+	botID string
+	*bridge.Config
+}
+
+func New(cfg *bridge.Config) bridge.Bridger {
+	return &Bmsteams{Config: cfg}
+}
+
+func (b *Bmsteams) Connect() error {
+	tokenCachePath := b.GetString("sessionFile")
+	if tokenCachePath == "" {
+		tokenCachePath = "msteams_session.json"
+	}
+	ctx := context.Background()
+	m := msauth.NewManager()
+	m.LoadFile(tokenCachePath) //nolint:errcheck
+	ts, err := m.DeviceAuthorizationGrant(ctx, b.GetString("TenantID"), b.GetString("ClientID"), defaultScopes, nil)
+	if err != nil {
+		return err
+	}
+	err = m.SaveFile(tokenCachePath)
+	if err != nil {
+		b.Log.Errorf("Couldn't save sessionfile in %s: %s", tokenCachePath, err)
+	}
+	// make file readable only for matterbridge user
+	err = os.Chmod(tokenCachePath, 0600)
+	if err != nil {
+		b.Log.Errorf("Couldn't change permissions for %s: %s", tokenCachePath, err)
+	}
+	httpClient := oauth2.NewClient(ctx, ts)
+	graphClient := msgraph.NewClient(httpClient)
+	b.gc = graphClient
+	b.ctx = ctx
+
+	err = b.setBotID()
+	if err != nil {
+		return err
+	}
+	b.Log.Info("Connection succeeded")
+	return nil
+}
+
+func (b *Bmsteams) Disconnect() error {
+	return nil
+}
+
+func (b *Bmsteams) JoinChannel(channel config.ChannelInfo) error {
+	go func(name string) {
+		for {
+			err := b.poll(name)
+			if err != nil {
+				b.Log.Errorf("polling failed for %s: %s. retrying in 5 seconds", name, err)
+			}
+			time.Sleep(time.Second * 5)
+		}
+	}(channel.Name)
+	return nil
+}
+
+func (b *Bmsteams) Send(msg config.Message) (string, error) {
+	b.Log.Debugf("=> Receiving %#v", msg)
+	if msg.ParentValid() {
+		return b.sendReply(msg)
+	}
+
+	// Handle prefix hint for unthreaded messages.
+	if msg.ParentNotFound() {
+		msg.ParentID = ""
+		msg.Text = fmt.Sprintf("[thread]: %s", msg.Text)
+	}
+
+	ct := b.gc.Teams().ID(b.GetString("TeamID")).Channels().ID(msg.Channel).Messages().Request()
+	text := msg.Username + msg.Text
+	content := &msgraph.ItemBody{Content: &text}
+	rmsg := &msgraph.ChatMessage{Body: content}
+	res, err := ct.Add(b.ctx, rmsg)
+	if err != nil {
+		return "", err
+	}
+	return *res.ID, nil
+}
+
+func (b *Bmsteams) sendReply(msg config.Message) (string, error) {
+	ct := b.gc.Teams().ID(b.GetString("TeamID")).Channels().ID(msg.Channel).Messages().ID(msg.ParentID).Replies().Request()
+	// Handle prefix hint for unthreaded messages.
+
+	text := msg.Username + msg.Text
+	content := &msgraph.ItemBody{Content: &text}
+	rmsg := &msgraph.ChatMessage{Body: content}
+	res, err := ct.Add(b.ctx, rmsg)
+	if err != nil {
+		return "", err
+	}
+	return *res.ID, nil
+}
+
+func (b *Bmsteams) getMessages(channel string) ([]msgraph.ChatMessage, error) {
+	ct := b.gc.Teams().ID(b.GetString("TeamID")).Channels().ID(channel).Messages().Request()
+	rct, err := ct.Get(b.ctx)
+	if err != nil {
+		return nil, err
+	}
+	b.Log.Debugf("got %#v messages", len(rct))
+	return rct, nil
+}
+
+//nolint:gocognit
+func (b *Bmsteams) poll(channelName string) error {
+	msgmap := make(map[string]time.Time)
+	b.Log.Debug("getting initial messages")
+	res, err := b.getMessages(channelName)
+	if err != nil {
+		return err
+	}
+	for _, msg := range res {
+		msgmap[*msg.ID] = *msg.CreatedDateTime
+		if msg.LastModifiedDateTime != nil {
+			msgmap[*msg.ID] = *msg.LastModifiedDateTime
+		}
+	}
+	time.Sleep(time.Second * 5)
+	b.Log.Debug("polling for messages")
+	for {
+		res, err := b.getMessages(channelName)
+		if err != nil {
+			return err
+		}
+		for i := len(res) - 1; i >= 0; i-- {
+			msg := res[i]
+			if mtime, ok := msgmap[*msg.ID]; ok {
+				if mtime == *msg.CreatedDateTime && msg.LastModifiedDateTime == nil {
+					continue
+				}
+				if msg.LastModifiedDateTime != nil && mtime == *msg.LastModifiedDateTime {
+					continue
+				}
+			}
+
+			if b.GetBool("debug") {
+				b.Log.Debug("Msg dump: ", spew.Sdump(msg))
+			}
+
+			// skip non-user message for now.
+			if msg.From.User == nil {
+				continue
+			}
+
+			if *msg.From.User.ID == b.botID {
+				b.Log.Debug("skipping own message")
+				msgmap[*msg.ID] = *msg.CreatedDateTime
+				continue
+			}
+
+			msgmap[*msg.ID] = *msg.CreatedDateTime
+			if msg.LastModifiedDateTime != nil {
+				msgmap[*msg.ID] = *msg.LastModifiedDateTime
+			}
+			b.Log.Debugf("<= Sending message from %s on %s to gateway", *msg.From.User.DisplayName, b.Account)
+			text := b.convertToMD(*msg.Body.Content)
+			rmsg := config.Message{
+				Username: *msg.From.User.DisplayName,
+				Text:     text,
+				Channel:  channelName,
+				Account:  b.Account,
+				Avatar:   "",
+				UserID:   *msg.From.User.ID,
+				ID:       *msg.ID,
+				Extra:    make(map[string][]interface{}),
+			}
+
+			b.handleAttachments(&rmsg, msg)
+			b.Log.Debugf("<= Message is %#v", rmsg)
+			b.Remote <- rmsg
+		}
+		time.Sleep(time.Second * 5)
+	}
+}
+
+func (b *Bmsteams) setBotID() error {
+	req := b.gc.Me().Request()
+	r, err := req.Get(b.ctx)
+	if err != nil {
+		return err
+	}
+	b.botID = *r.ID
+	return nil
+}
+
+func (b *Bmsteams) convertToMD(text string) string {
+	if !strings.Contains(text, "<div>") {
+		return text
+	}
+	var sb strings.Builder
+	err := godown.Convert(&sb, strings.NewReader(text), nil)
+	if err != nil {
+		b.Log.Errorf("Couldn't convert message to markdown %s", text)
+		return text
+	}
+	return sb.String()
+}

bridge/mumble/handlers.go

@@ -0,0 +1,96 @@
+package bmumble
+
+import (
+	"strconv"
+	"time"
+
+	"layeh.com/gumble/gumble"
+
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/42wim/matterbridge/bridge/helper"
+)
+
+func (b *Bmumble) handleServerConfig(event *gumble.ServerConfigEvent) {
+	b.serverConfigUpdate <- *event
+}
+
+func (b *Bmumble) handleTextMessage(event *gumble.TextMessageEvent) {
+	sender := "unknown"
+	if event.TextMessage.Sender != nil {
+		sender = event.TextMessage.Sender.Name
+	}
+	// If the text message is received before receiving a ServerSync
+	// and UserState, Client.Self or Self.Channel are nil
+	if event.Client.Self == nil || event.Client.Self.Channel == nil {
+		b.Log.Warn("Connection bootstrap not finished, discarding text message")
+		return
+	}
+	// Convert Mumble HTML messages to markdown
+	parts, err := b.convertHTMLtoMarkdown(event.TextMessage.Message)
+	if err != nil {
+		b.Log.Error(err)
+	}
+	now := time.Now().UTC()
+	for i, part := range parts {
+		// Construct matterbridge message and pass on to the gateway
+		rmsg := config.Message{
+			Channel:  strconv.FormatUint(uint64(event.Client.Self.Channel.ID), 10),
+			Username: sender,
+			UserID:   sender + "@" + b.Host,
+			Account:  b.Account,
+		}
+		if part.Image == nil {
+			rmsg.Text = part.Text
+		} else {
+			fname := b.Account + "_" + strconv.FormatInt(now.UnixNano(), 10) + "_" + strconv.Itoa(i) + part.FileExtension
+			rmsg.Extra = make(map[string][]interface{})
+			if err = helper.HandleDownloadSize(b.Log, &rmsg, fname, int64(len(part.Image)), b.General); err != nil {
+				b.Log.WithError(err).Warn("not including image in message")
+				continue
+			}
+			helper.HandleDownloadData(b.Log, &rmsg, fname, "", "", &part.Image, b.General)
+		}
+		b.Log.Debugf("Sending message to gateway: %+v", rmsg)
+		b.Remote <- rmsg
+	}
+}
+
+func (b *Bmumble) handleConnect(event *gumble.ConnectEvent) {
+	// Set the user's "bio"/comment
+	if comment := b.GetString("UserComment"); comment != "" && event.Client.Self != nil {
+		event.Client.Self.SetComment(comment)
+	}
+	// No need to talk or listen
+	event.Client.Self.SetSelfDeafened(true)
+	event.Client.Self.SetSelfMuted(true)
+	// if the Channel variable is set, this is a reconnect -> rejoin channel
+	if b.Channel != nil {
+		if err := b.doJoin(event.Client, *b.Channel); err != nil {
+			b.Log.Error(err)
+		}
+		b.Remote <- config.Message{
+			Username: "system",
+			Text:     "rejoin",
+			Channel:  "",
+			Account:  b.Account,
+			Event:    config.EventRejoinChannels,
+		}
+	}
+}
+
+func (b *Bmumble) handleUserChange(event *gumble.UserChangeEvent) {
+	// Only care about changes to self
+	if event.User != event.Client.Self {
+		return
+	}
+	// Someone attempted to move the user out of the configured channel; attempt to join back
+	if b.Channel != nil {
+		if err := b.doJoin(event.Client, *b.Channel); err != nil {
+			b.Log.Error(err)
+		}
+	}
+}
+
+func (b *Bmumble) handleDisconnect(event *gumble.DisconnectEvent) {
+	b.connected <- *event
+}

bridge/mumble/helpers.go

@@ -0,0 +1,143 @@
+package bmumble
+
+import (
+	"fmt"
+	"mime"
+	"net/http"
+	"regexp"
+	"strings"
+
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/mattn/godown"
+	"github.com/vincent-petithory/dataurl"
+)
+
+type MessagePart struct {
+	Text          string
+	FileExtension string
+	Image         []byte
+}
+
+func (b *Bmumble) decodeImage(uri string, parts *[]MessagePart) error {
+	// Decode the data:image/... URI
+	image, err := dataurl.DecodeString(uri)
+	if err != nil {
+		b.Log.WithError(err).Info("No image extracted")
+		return err
+	}
+	// Determine the file extensions for that image
+	ext, err := mime.ExtensionsByType(image.MediaType.ContentType())
+	if err != nil || len(ext) == 0 {
+		b.Log.WithError(err).Infof("No file extension registered for MIME type '%s'", image.MediaType.ContentType())
+		return err
+	}
+	// Add the image to the MessagePart slice
+	*parts = append(*parts, MessagePart{"", ext[0], image.Data})
+	return nil
+}
+
+func (b *Bmumble) tokenize(t *string) ([]MessagePart, error) {
+	// `^(.*?)` matches everything before the image
+	// `!\[[^\]]*\]\(` matches the `![alt](` part of markdown images
+	// `(data:image\/[^)]+)` matches the data: URI used by Mumble
+	// `\)` matches the closing parenthesis after the URI
+	// `(.*)$` matches the remaining text to be examined in the next iteration
+	p := regexp.MustCompile(`^(?ms)(.*?)!\[[^\]]*\]\((data:image\/[^)]+)\)(.*)$`)
+	remaining := *t
+	var parts []MessagePart
+	for {
+		tokens := p.FindStringSubmatch(remaining)
+		if tokens == nil {
+			// no match -> remaining string is non-image text
+			pre := strings.TrimSpace(remaining)
+			if len(pre) > 0 {
+				parts = append(parts, MessagePart{pre, "", nil})
+			}
+			return parts, nil
+		}
+
+		// tokens[1] is the text before the image
+		if len(tokens[1]) > 0 {
+			pre := strings.TrimSpace(tokens[1])
+			parts = append(parts, MessagePart{pre, "", nil})
+		}
+		// tokens[2] is the image URL
+		uri, err := dataurl.UnescapeToString(strings.TrimSpace(strings.ReplaceAll(tokens[2], " ", "")))
+		if err != nil {
+			b.Log.WithError(err).Info("URL unescaping failed")
+			remaining = strings.TrimSpace(tokens[3])
+			continue
+		}
+		err = b.decodeImage(uri, &parts)
+		if err != nil {
+			b.Log.WithError(err).Info("Decoding the image failed")
+		}
+		// tokens[3] is the text after the image, processed in the next iteration
+		remaining = strings.TrimSpace(tokens[3])
+	}
+}
+
+func (b *Bmumble) convertHTMLtoMarkdown(html string) ([]MessagePart, error) {
+	var sb strings.Builder
+	err := godown.Convert(&sb, strings.NewReader(html), nil)
+	if err != nil {
+		return nil, err
+	}
+	markdown := sb.String()
+	b.Log.Debugf("### to markdown: %s", markdown)
+	return b.tokenize(&markdown)
+}
+
+func (b *Bmumble) extractFiles(msg *config.Message) []config.Message {
+	var messages []config.Message
+	if msg.Extra == nil || len(msg.Extra["file"]) == 0 {
+		return messages
+	}
+	// Create a separate message for each file
+	for _, f := range msg.Extra["file"] {
+		fi := f.(config.FileInfo)
+		imsg := config.Message{
+			Channel:   msg.Channel,
+			Username:  msg.Username,
+			UserID:    msg.UserID,
+			Account:   msg.Account,
+			Protocol:  msg.Protocol,
+			Timestamp: msg.Timestamp,
+			Event:     "mumble_image",
+		}
+		// If no data is present for the file, send a link instead
+		if fi.Data == nil || len(*fi.Data) == 0 {
+			if len(fi.URL) > 0 {
+				imsg.Text = fmt.Sprintf(`<a href="%s">%s</a>`, fi.URL, fi.URL)
+				messages = append(messages, imsg)
+			} else {
+				b.Log.Infof("Not forwarding file without local data")
+			}
+			continue
+		}
+		mimeType := http.DetectContentType(*fi.Data)
+		// Mumble only supports images natively, send a link instead
+		if !strings.HasPrefix(mimeType, "image/") {
+			if len(fi.URL) > 0 {
+				imsg.Text = fmt.Sprintf(`<a href="%s">%s</a>`, fi.URL, fi.URL)
+				messages = append(messages, imsg)
+			} else {
+				b.Log.Infof("Not forwarding file of type %s", mimeType)
+			}
+			continue
+		}
+		mimeType = strings.TrimSpace(strings.Split(mimeType, ";")[0])
+		// Build data:image/...;base64,... style image URL and embed image directly into the message
+		du := dataurl.New(*fi.Data, mimeType)
+		dataURL, err := du.MarshalText()
+		if err != nil {
+			b.Log.WithError(err).Infof("Image Serialization into data URL failed (type: %s, length: %d)", mimeType, len(*fi.Data))
+			continue
+		}
+		imsg.Text = fmt.Sprintf(`<img src="%s"/>`, dataURL)
+		messages = append(messages, imsg)
+	}
+	// Remove files from original message
+	msg.Extra["file"] = nil
+	return messages
+}

bridge/mumble/mumble.go

@@ -0,0 +1,259 @@
+package bmumble
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"strconv"
+	"time"
+
+	"layeh.com/gumble/gumble"
+	"layeh.com/gumble/gumbleutil"
+
+	"github.com/42wim/matterbridge/bridge"
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/42wim/matterbridge/bridge/helper"
+	stripmd "github.com/writeas/go-strip-markdown"
+
+	// We need to import the 'data' package as an implicit dependency.
+	// See: https://godoc.org/github.com/paulrosania/go-charset/charset
+	_ "github.com/paulrosania/go-charset/data"
+)
+
+type Bmumble struct {
+	client             *gumble.Client
+	Nick               string
+	Host               string
+	Channel            *uint32
+	local              chan config.Message
+	running            chan error
+	connected          chan gumble.DisconnectEvent
+	serverConfigUpdate chan gumble.ServerConfigEvent
+	serverConfig       gumble.ServerConfigEvent
+	tlsConfig          tls.Config
+
+	*bridge.Config
+}
+
+func New(cfg *bridge.Config) bridge.Bridger {
+	b := &Bmumble{}
+	b.Config = cfg
+	b.Nick = b.GetString("Nick")
+	b.local = make(chan config.Message)
+	b.running = make(chan error)
+	b.connected = make(chan gumble.DisconnectEvent)
+	b.serverConfigUpdate = make(chan gumble.ServerConfigEvent)
+	return b
+}
+
+func (b *Bmumble) Connect() error {
+	b.Log.Infof("Connecting %s", b.GetString("Server"))
+	host, portstr, err := net.SplitHostPort(b.GetString("Server"))
+	if err != nil {
+		return err
+	}
+	b.Host = host
+	_, err = strconv.Atoi(portstr)
+	if err != nil {
+		return err
+	}
+
+	if err = b.buildTLSConfig(); err != nil {
+		return err
+	}
+
+	go b.doSend()
+	go b.connectLoop()
+	err = <-b.running
+	return err
+}
+
+func (b *Bmumble) Disconnect() error {
+	return b.client.Disconnect()
+}
+
+func (b *Bmumble) JoinChannel(channel config.ChannelInfo) error {
+	cid, err := strconv.ParseUint(channel.Name, 10, 32)
+	if err != nil {
+		return err
+	}
+	channelID := uint32(cid)
+	if b.Channel != nil && *b.Channel != channelID {
+		b.Log.Fatalf("Cannot join channel ID '%d', already joined to channel ID %d", channelID, *b.Channel)
+		return errors.New("the Mumble bridge can only join a single channel")
+	}
+	b.Channel = &channelID
+	return b.doJoin(b.client, channelID)
+}
+
+func (b *Bmumble) Send(msg config.Message) (string, error) {
+	// Only process text messages
+	b.Log.Debugf("=> Received local message %#v", msg)
+	if msg.Event != "" && msg.Event != config.EventUserAction {
+		return "", nil
+	}
+
+	attachments := b.extractFiles(&msg)
+	b.local <- msg
+	for _, a := range attachments {
+		b.local <- a
+	}
+	return "", nil
+}
+
+func (b *Bmumble) buildTLSConfig() error {
+	b.tlsConfig = tls.Config{}
+	// Load TLS client certificate keypair required for registered user authentication
+	if cpath := b.GetString("TLSClientCertificate"); cpath != "" {
+		if ckey := b.GetString("TLSClientKey"); ckey != "" {
+			cert, err := tls.LoadX509KeyPair(cpath, ckey)
+			if err != nil {
+				return err
+			}
+			b.tlsConfig.Certificates = []tls.Certificate{cert}
+		}
+	}
+	// Load TLS CA used for server verification.  If not provided, the Go system trust anchor is used
+	if capath := b.GetString("TLSCACertificate"); capath != "" {
+		ca, err := ioutil.ReadFile(capath)
+		if err != nil {
+			return err
+		}
+		b.tlsConfig.RootCAs = x509.NewCertPool()
+		b.tlsConfig.RootCAs.AppendCertsFromPEM(ca)
+	}
+	b.tlsConfig.InsecureSkipVerify = b.GetBool("SkipTLSVerify")
+	return nil
+}
+
+func (b *Bmumble) connectLoop() {
+	firstConnect := true
+	for {
+		err := b.doConnect()
+		if firstConnect {
+			b.running <- err
+		}
+		if err != nil {
+			b.Log.Errorf("Connection to server failed: %#v", err)
+			if firstConnect {
+				break
+			} else {
+				b.Log.Info("Retrying in 10s")
+				time.Sleep(10 * time.Second)
+				continue
+			}
+		}
+		firstConnect = false
+		d := <-b.connected
+		switch d.Type {
+		case gumble.DisconnectError:
+			b.Log.Errorf("Lost connection to the server (%s), attempting reconnect", d.String)
+			continue
+		case gumble.DisconnectKicked:
+			b.Log.Errorf("Kicked from the server (%s), attempting reconnect", d.String)
+			continue
+		case gumble.DisconnectBanned:
+			b.Log.Errorf("Banned from the server (%s), not attempting reconnect", d.String)
+			close(b.connected)
+			close(b.running)
+			return
+		case gumble.DisconnectUser:
+			b.Log.Infof("Disconnect successful")
+			close(b.connected)
+			close(b.running)
+			return
+		}
+	}
+}
+
+func (b *Bmumble) doConnect() error {
+	// Create new gumble config and attach event handlers
+	gumbleConfig := gumble.NewConfig()
+	gumbleConfig.Attach(gumbleutil.Listener{
+		ServerConfig: b.handleServerConfig,
+		TextMessage:  b.handleTextMessage,
+		Connect:      b.handleConnect,
+		Disconnect:   b.handleDisconnect,
+		UserChange:   b.handleUserChange,
+	})
+	gumbleConfig.Username = b.GetString("Nick")
+	if password := b.GetString("Password"); password != "" {
+		gumbleConfig.Password = password
+	}
+
+	client, err := gumble.DialWithDialer(new(net.Dialer), b.GetString("Server"), gumbleConfig, &b.tlsConfig)
+	if err != nil {
+		return err
+	}
+	b.client = client
+	return nil
+}
+
+func (b *Bmumble) doJoin(client *gumble.Client, channelID uint32) error {
+	channel, ok := client.Channels[channelID]
+	if !ok {
+		return fmt.Errorf("no channel with ID %d", channelID)
+	}
+	client.Self.Move(channel)
+	return nil
+}
+
+func (b *Bmumble) doSend() {
+	// Message sending loop that makes sure server-side
+	// restrictions and client-side message traits don't conflict
+	// with each other.
+	for {
+		select {
+		case serverConfig := <-b.serverConfigUpdate:
+			b.Log.Debugf("Received server config update: AllowHTML=%#v, MaximumMessageLength=%#v", serverConfig.AllowHTML, serverConfig.MaximumMessageLength)
+			b.serverConfig = serverConfig
+		case msg := <-b.local:
+			b.processMessage(&msg)
+		}
+	}
+}
+
+func (b *Bmumble) processMessage(msg *config.Message) {
+	b.Log.Debugf("Processing message %s", msg.Text)
+
+	allowHTML := true
+	if b.serverConfig.AllowHTML != nil {
+		allowHTML = *b.serverConfig.AllowHTML
+	}
+
+	// If this is a specially generated image message, send it unmodified
+	if msg.Event == "mumble_image" {
+		if allowHTML {
+			b.client.Self.Channel.Send(msg.Username+msg.Text, false)
+		} else {
+			b.Log.Info("Can't send image, server does not allow HTML messages")
+		}
+		return
+	}
+
+	// Don't process empty messages
+	if len(msg.Text) == 0 {
+		return
+	}
+	// If HTML is allowed, convert markdown into HTML, otherwise strip markdown
+	if allowHTML {
+		msg.Text = helper.ParseMarkdown(msg.Text)
+	} else {
+		msg.Text = stripmd.Strip(msg.Text)
+	}
+
+	// If there is a maximum message length, split and truncate the lines
+	var msgLines []string
+	if maxLength := b.serverConfig.MaximumMessageLength; maxLength != nil {
+		msgLines = helper.GetSubLines(msg.Text, *maxLength-len(msg.Username))
+	} else {
+		msgLines = helper.GetSubLines(msg.Text, 0)
+	}
+	// Send the individual lindes
+	for i := range msgLines {
+		b.client.Self.Channel.Send(msg.Username+msgLines[i], false)
+	}
+}

bridge/nctalk/nctalk.go

@@ -0,0 +1,210 @@
+package nctalk
+
+import (
+	"context"
+	"crypto/tls"
+	"strconv"
+	"strings"
+
+	"github.com/42wim/matterbridge/bridge"
+	"github.com/42wim/matterbridge/bridge/config"
+
+	"gomod.garykim.dev/nc-talk/ocs"
+	"gomod.garykim.dev/nc-talk/room"
+	"gomod.garykim.dev/nc-talk/user"
+)
+
+type Btalk struct {
+	user  *user.TalkUser
+	rooms []Broom
+	*bridge.Config
+}
+
+func New(cfg *bridge.Config) bridge.Bridger {
+	return &Btalk{Config: cfg}
+}
+
+type Broom struct {
+	room      *room.TalkRoom
+	ctx       context.Context
+	ctxCancel context.CancelFunc
+}
+
+func (b *Btalk) Connect() error {
+	b.Log.Info("Connecting")
+	tconfig := &user.TalkUserConfig{
+		TLSConfig: &tls.Config{
+			InsecureSkipVerify: b.GetBool("SkipTLSVerify"), //nolint:gosec
+		},
+	}
+	var err error
+	b.user, err = user.NewUser(b.GetString("Server"), b.GetString("Login"), b.GetString("Password"), tconfig)
+	if err != nil {
+		b.Log.Error("Config could not be used")
+		return err
+	}
+	_, err = b.user.Capabilities()
+	if err != nil {
+		b.Log.Error("Cannot Connect")
+		return err
+	}
+	b.Log.Info("Connected")
+	return nil
+}
+
+func (b *Btalk) Disconnect() error {
+	for _, r := range b.rooms {
+		r.ctxCancel()
+	}
+	return nil
+}
+
+func (b *Btalk) JoinChannel(channel config.ChannelInfo) error {
+	tr, err := room.NewTalkRoom(b.user, channel.Name)
+	if err != nil {
+		return err
+	}
+	newRoom := Broom{
+		room: tr,
+	}
+	newRoom.ctx, newRoom.ctxCancel = context.WithCancel(context.Background())
+	c, err := newRoom.room.ReceiveMessages(newRoom.ctx)
+	if err != nil {
+		return err
+	}
+	b.rooms = append(b.rooms, newRoom)
+
+	// Config
+	guestSuffix := " (Guest)"
+	if b.IsKeySet("GuestSuffix") {
+		guestSuffix = b.GetString("GuestSuffix")
+	}
+
+	go func() {
+		for msg := range c {
+			msg := msg
+
+			if msg.Error != nil {
+				b.Log.Errorf("Fatal message poll error: %s\n", msg.Error)
+
+				return
+			}
+
+			// ignore messages that are one of the following
+			// * not a message from a user
+			// * from ourselves
+			if msg.MessageType != ocs.MessageComment || msg.ActorID == b.user.User {
+				continue
+			}
+			remoteMessage := config.Message{
+				Text:     formatRichObjectString(msg.Message, msg.MessageParameters),
+				Channel:  newRoom.room.Token,
+				Username: DisplayName(msg, guestSuffix),
+				UserID:   msg.ActorID,
+				Account:  b.Account,
+			}
+			// It is possible for the ID to not be set on older versions of Talk so we only set it if
+			// the ID is not blank
+			if msg.ID != 0 {
+				remoteMessage.ID = strconv.Itoa(msg.ID)
+			}
+
+			// Handle Files
+			err = b.handleFiles(&remoteMessage, &msg)
+			if err != nil {
+				b.Log.Errorf("Error handling file: %#v", msg)
+
+				continue
+			}
+
+			b.Log.Debugf("<= Message is %#v", remoteMessage)
+			b.Remote <- remoteMessage
+		}
+	}()
+	return nil
+}
+
+func (b *Btalk) Send(msg config.Message) (string, error) {
+	r := b.getRoom(msg.Channel)
+	if r == nil {
+		b.Log.Errorf("Could not find room for %v", msg.Channel)
+		return "", nil
+	}
+
+	// Talk currently only supports sending normal messages
+	if msg.Event != "" {
+		return "", nil
+	}
+	sentMessage, err := r.room.SendMessage(msg.Username + msg.Text)
+	if err != nil {
+		b.Log.Errorf("Could not send message to room %v from %v: %v", msg.Channel, msg.Username, err)
+		return "", nil
+	}
+	return strconv.Itoa(sentMessage.ID), nil
+}
+
+func (b *Btalk) getRoom(token string) *Broom {
+	for _, r := range b.rooms {
+		if r.room.Token == token {
+			return &r
+		}
+	}
+	return nil
+}
+
+func (b *Btalk) handleFiles(mmsg *config.Message, message *ocs.TalkRoomMessageData) error {
+	for _, parameter := range message.MessageParameters {
+		if parameter.Type == ocs.ROSTypeFile {
+			// Get the file
+			file, err := b.user.DownloadFile(parameter.Path)
+			if err != nil {
+				return err
+			}
+
+			if mmsg.Extra == nil {
+				mmsg.Extra = make(map[string][]interface{})
+			}
+
+			mmsg.Extra["file"] = append(mmsg.Extra["file"], config.FileInfo{
+				Name:   parameter.Name,
+				Data:   file,
+				Size:   int64(len(*file)),
+				Avatar: false,
+			})
+		}
+	}
+
+	return nil
+}
+
+// Spec: https://github.com/nextcloud/server/issues/1706#issue-182308785
+func formatRichObjectString(message string, parameters map[string]ocs.RichObjectString) string {
+	for id, parameter := range parameters {
+		text := parameter.Name
+
+		switch parameter.Type {
+		case ocs.ROSTypeUser, ocs.ROSTypeGroup:
+			text = "@" + text
+		case ocs.ROSTypeFile:
+			if parameter.Link != "" {
+				text = parameter.Name
+			}
+		}
+
+		message = strings.ReplaceAll(message, "{"+id+"}", text)
+	}
+
+	return message
+}
+
+func DisplayName(msg ocs.TalkRoomMessageData, suffix string) string {
+	if msg.ActorType == ocs.ActorGuest {
+		if msg.ActorDisplayName == "" {
+			return "Guest"
+		}
+
+		return msg.ActorDisplayName + suffix
+	}
+
+	return msg.ActorDisplayName
+}

bridge/rocketchat/handlers.go

@@ -2,6 +2,7 @@ package brocketchat
 
 import (
 	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/matterbridge/Rocket.Chat.Go.SDK/models"
 )
 
 func (b *Brocketchat) handleRocket() {
@@ -38,6 +39,23 @@ func (b *Brocketchat) handleRocketHook(messages chan *config.Message) {
 	}
 }
 
+func (b *Brocketchat) handleStatusEvent(ev models.Message, rmsg *config.Message) bool {
+	switch ev.Type {
+	case "":
+		// this is a normal message, no processing needed
+		// return true so the message is not dropped
+		return true
+	case sUserJoined, sUserLeft:
+		rmsg.Event = config.EventJoinLeave
+		return true
+	case sRoomChangedTopic:
+		rmsg.Event = config.EventTopicChange
+		return true
+	}
+	b.Log.Debugf("Dropping message with unknown type: %s", ev.Type)
+	return false
+}
+
 func (b *Brocketchat) handleRocketClient(messages chan *config.Message) {
 	for message := range b.messageChan {
 		// skip messages with same ID, apparently messages get duplicated for an unknown reason
@@ -59,7 +77,12 @@ func (b *Brocketchat) handleRocketClient(messages chan *config.Message) {
 			UserID:   message.User.ID,
 			ID:       message.ID,
 		}
-		messages <- rmsg
+
+		// handleStatusEvent returns false if the message should be dropped
+		// in that case it is probably some modification to the channel we do not want to relay
+		if b.handleStatusEvent(m, rmsg) {
+			messages <- rmsg
+		}
 	}
 }
 

bridge/rocketchat/rocketchat.go

@@ -29,6 +29,12 @@ type Brocketchat struct {
 	sync.RWMutex
 }
 
+const (
+	sUserJoined       = "uj"
+	sUserLeft         = "ul"
+	sRoomChangedTopic = "room_changed_topic"
+)
+
 func New(cfg *bridge.Config) bridge.Bridger {
 	newCache, err := lru.New(100)
 	if err != nil {

bridge/slack/handlers.go

@@ -1,18 +1,22 @@
 package bslack
 
 import (
+	"errors"
 	"fmt"
 	"html"
 	"time"
 
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/42wim/matterbridge/bridge/helper"
-	"github.com/nlopes/slack"
+	"github.com/slack-go/slack"
 )
 
+// ErrEventIgnored is for events that should be ignored
+var ErrEventIgnored = errors.New("this event message should ignored")
+
 func (b *Bslack) handleSlack() {
 	messages := make(chan *config.Message)
-	if b.GetString(incomingWebhookConfig) != "" {
+	if b.GetString(incomingWebhookConfig) != "" && b.GetString(tokenConfig) == "" {
 		b.Log.Debugf("Choosing webhooks based receiving")
 		go b.handleMatterHook(messages)
 	} else {
@@ -53,7 +57,9 @@ func (b *Bslack) handleSlackClient(messages chan *config.Message) {
 				continue
 			}
 			rmsg, err := b.handleTypingEvent(ev)
-			if err != nil {
+			if err == ErrEventIgnored {
+				continue
+			} else if err != nil {
 				b.Log.Errorf("%#v", err)
 				continue
 			}
@@ -87,7 +93,7 @@ func (b *Bslack) handleSlackClient(messages chan *config.Message) {
 			b.Log.Errorf("Connection failed %#v %#v", ev.Error(), ev.ErrorObj)
 		case *slack.MemberJoinedChannelEvent:
 			b.users.populateUser(ev.User)
-		case *slack.HelloEvent, *slack.LatencyReport:
+		case *slack.HelloEvent, *slack.LatencyReport, *slack.ConnectingEvent:
 			continue
 		default:
 			b.Log.Debugf("Unhandled incoming event: %T", ev)
@@ -124,11 +130,11 @@ func (b *Bslack) skipMessageEvent(ev *slack.MessageEvent) bool {
 		}
 	}
 
-	// Skip any messages that we made ourselves or from 'slackbot' (see #527).
+	// Check for our callback ID
-	if ev.Username == sSlackBotUser ||
+	hasOurCallbackID := false
-		(b.rtm != nil && ev.Username == b.si.User.Name) ||
+	if len(ev.Blocks.BlockSet) == 1 {
-		(len(ev.Attachments) > 0 && ev.Attachments[0].CallbackID == "matterbridge_"+b.uuid) {
+		block, ok := ev.Blocks.BlockSet[0].(*slack.SectionBlock)
-		return true
+		hasOurCallbackID = ok && block.BlockID == "matterbridge_"+b.uuid
 	}
 
 	if ev.SubMessage != nil {
@@ -143,6 +149,16 @@ func (b *Bslack) skipMessageEvent(ev *slack.MessageEvent) bool {
 		if ev.SubType == "message_replied" && ev.Hidden {
 			return true
 		}
+		if len(ev.SubMessage.Blocks.BlockSet) == 1 {
+			block, ok := ev.SubMessage.Blocks.BlockSet[0].(*slack.SectionBlock)
+			hasOurCallbackID = ok && block.BlockID == "matterbridge_"+b.uuid
+		}
+	}
+
+	// Skip any messages that we made ourselves or from 'slackbot' (see #527).
+	if ev.Username == sSlackBotUser ||
+		(b.rtm != nil && ev.Username == b.si.User.Name) || hasOurCallbackID {
+		return true
 	}
 
 	if len(ev.Files) > 0 {
@@ -270,6 +286,9 @@ func (b *Bslack) handleAttachments(ev *slack.MessageEvent, rmsg *config.Message)
 }
 
 func (b *Bslack) handleTypingEvent(ev *slack.UserTypingEvent) (*config.Message, error) {
+	if ev.User == b.si.User.ID {
+		return nil, ErrEventIgnored
+	}
 	channelInfo, err := b.channels.getChannelByID(ev.Channel)
 	if err != nil {
 		return nil, err

bridge/slack/helpers.go

@@ -7,8 +7,8 @@ import (
 	"time"
 
 	"github.com/42wim/matterbridge/bridge/config"
-	"github.com/nlopes/slack"
 	"github.com/sirupsen/logrus"
+	"github.com/slack-go/slack"
 )
 
 // populateReceivedMessage shapes the initial Matterbridge message that we will forward to the

bridge/slack/legacy.go

@@ -5,7 +5,7 @@ import (
 
 	"github.com/42wim/matterbridge/bridge"
 	"github.com/42wim/matterbridge/matterhook"
-	"github.com/nlopes/slack"
+	"github.com/slack-go/slack"
 )
 
 type BLegacy struct {

bridge/slack/slack.go

@@ -13,8 +13,8 @@ import (
 	"github.com/42wim/matterbridge/bridge/helper"
 	"github.com/42wim/matterbridge/matterhook"
 	lru "github.com/hashicorp/golang-lru"
-	"github.com/nlopes/slack"
 	"github.com/rs/xid"
+	"github.com/slack-go/slack"
 )
 
 type Bslack struct {
@@ -64,6 +64,7 @@ const (
 	editSuffixConfig      = "EditSuffix"
 	iconURLConfig         = "iconurl"
 	noSendJoinConfig      = "nosendjoinpart"
+	messageLength         = 3000
 )
 
 func New(cfg *bridge.Config) bridge.Bridger {
@@ -194,6 +195,7 @@ func (b *Bslack) Send(msg config.Message) (string, error) {
 		b.Log.Debugf("=> Receiving %#v", msg)
 	}
 
+	msg.Text = helper.ClipMessage(msg.Text, messageLength)
 	msg.Text = b.replaceCodeFence(msg.Text)
 
 	// Make a action /me of the message
@@ -202,7 +204,7 @@ func (b *Bslack) Send(msg config.Message) (string, error) {
 	}
 
 	// Use webhook to send the message
-	if b.GetString(outgoingWebhookConfig) != "" {
+	if b.GetString(outgoingWebhookConfig) != "" && b.GetString(tokenConfig) == "" {
 		return "", b.sendWebhook(msg)
 	}
 	return b.sendRTM(msg)
@@ -297,7 +299,7 @@ func (b *Bslack) sendRTM(msg config.Message) (string, error) {
 	}
 
 	// Handle prefix hint for unthreaded messages.
-	if msg.ParentID == "msg-parent-not-found" {
+	if msg.ParentNotFound() {
 		msg.ParentID = ""
 		msg.Text = fmt.Sprintf("[thread]: %s", msg.Text)
 	}
@@ -408,7 +410,6 @@ func (b *Bslack) editMessage(msg *config.Message, channelInfo *slack.Channel) (b
 	}
 	messageOptions := b.prepareMessageOptions(msg)
 	for {
-		messageOptions = append(messageOptions, slack.MsgOptionText(msg.Text, false))
 		_, _, _, err := b.rtm.UpdateMessage(channelInfo.ID, msg.ID, messageOptions...)
 		if err == nil {
 			return true, nil
@@ -427,11 +428,6 @@ func (b *Bslack) postMessage(msg *config.Message, channelInfo *slack.Channel) (s
 		return "", nil
 	}
 	messageOptions := b.prepareMessageOptions(msg)
-	messageOptions = append(
-		messageOptions,
-		slack.MsgOptionText(msg.Text, false),
-		slack.MsgOptionEnableLinkUnfurl(),
-	)
 	for {
 		_, id, err := b.rtm.PostMessage(channelInfo.ID, messageOptions...)
 		if err == nil {
@@ -497,8 +493,6 @@ func (b *Bslack) prepareMessageOptions(msg *config.Message) []slack.MsgOption {
 	}
 
 	var attachments []slack.Attachment
-	// add a callback ID so we can see we created it
-	attachments = append(attachments, slack.Attachment{CallbackID: "matterbridge_" + b.uuid})
 	// add file attachments
 	attachments = append(attachments, b.createAttach(msg.Extra)...)
 	// add slack attachments (from another slack bridge)
@@ -509,6 +503,19 @@ func (b *Bslack) prepareMessageOptions(msg *config.Message) []slack.MsgOption {
 	}
 
 	var opts []slack.MsgOption
+	opts = append(opts,
+		// provide regular text field (fallback used in Slack notifications, etc.)
+		slack.MsgOptionText(msg.Text, false),
+
+		// add a callback ID so we can see we created it
+		slack.MsgOptionBlocks(slack.NewSectionBlock(
+			slack.NewTextBlockObject(slack.MarkdownType, msg.Text, false, false),
+			nil, nil,
+			slack.SectionBlockOptionBlockID("matterbridge_"+b.uuid),
+		)),
+
+		slack.MsgOptionEnableLinkUnfurl(),
+	)
 	opts = append(opts, slack.MsgOptionAttachments(attachments...))
 	opts = append(opts, slack.MsgOptionPostMessageParameters(params))
 	return opts

bridge/slack/users_channels.go

@@ -8,8 +8,8 @@ import (
 	"time"
 
 	"github.com/42wim/matterbridge/bridge/config"
-	"github.com/nlopes/slack"
 	"github.com/sirupsen/logrus"
+	"github.com/slack-go/slack"
 )
 
 const minimumRefreshInterval = 10 * time.Second

bridge/telegram/handlers.go

@@ -39,22 +39,32 @@ func (b *Btelegram) handleGroups(rmsg *config.Message, message *tgbotapi.Message
 
 // handleForwarded handles forwarded messages
 func (b *Btelegram) handleForwarded(rmsg *config.Message, message *tgbotapi.Message) {
-	if message.ForwardFrom != nil {
+	if message.ForwardDate == 0 {
-		usernameForward := ""
+		return
-		if b.GetBool("UseFirstName") {
+	}
+
+	if message.ForwardFrom == nil {
+		rmsg.Text = "Forwarded from " + unknownUser + ": " + rmsg.Text
+		return
+	}
+
+	usernameForward := ""
+	if b.GetBool("UseFirstName") {
+		usernameForward = message.ForwardFrom.FirstName
+	}
+
+	if usernameForward == "" {
+		usernameForward = message.ForwardFrom.UserName
+		if usernameForward == "" {
 			usernameForward = message.ForwardFrom.FirstName
 		}
-		if usernameForward == "" {
-			usernameForward = message.ForwardFrom.UserName
-			if usernameForward == "" {
-				usernameForward = message.ForwardFrom.FirstName
-			}
-		}
-		if usernameForward == "" {
-			usernameForward = unknownUser
-		}
-		rmsg.Text = "Forwarded from " + usernameForward + ": " + rmsg.Text
 	}
+
+	if usernameForward == "" {
+		usernameForward = unknownUser
+	}
+
+	rmsg.Text = "Forwarded from " + usernameForward + ": " + rmsg.Text
 }
 
 // handleQuoting handles quoting of previous messages
@@ -95,7 +105,7 @@ func (b *Btelegram) handleUsername(rmsg *config.Message, message *tgbotapi.Messa
 			}
 		}
 		// only download avatars if we have a place to upload them (configured mediaserver)
-		if b.General.MediaServerUpload != "" {
+		if b.General.MediaServerUpload != "" || (b.General.MediaServerDownload != "" && b.General.MediaDownloadPath != "") {
 			b.handleDownloadAvatar(message.From.ID, rmsg.Channel)
 		}
 	}
@@ -207,6 +217,46 @@ func (b *Btelegram) handleDownloadAvatar(userid int, channel string) {
 	}
 }
 
+func (b *Btelegram) maybeConvertTgs(name *string, data *[]byte) {
+	var format string
+	switch b.GetString("MediaConvertTgs") {
+	case FormatWebp:
+		b.Log.Debugf("Tgs to WebP conversion enabled, converting %v", name)
+		format = FormatWebp
+	case FormatPng:
+		// The WebP to PNG converter can't handle animated webp files yet,
+		// and I'm not going to write a path for x/image/webp.
+		// The error message would be:
+		//     conversion failed: webp: non-Alpha VP8X is not implemented
+		// So instead, we tell lottie to directly go to PNG.
+		b.Log.Debugf("Tgs to PNG conversion enabled, converting %v", name)
+		format = FormatPng
+	default:
+		// Otherwise, no conversion was requested. Trying to run the usual webp
+		// converter would fail, because '.tgs.webp' is actually a gzipped JSON
+		// file, and has nothing to do with WebP.
+		return
+	}
+	err := helper.ConvertTgsToX(data, format, b.Log)
+	if err != nil {
+		b.Log.Errorf("conversion failed: %v", err)
+	} else {
+		*name = strings.Replace(*name, "tgs.webp", format, 1)
+	}
+}
+
+func (b *Btelegram) maybeConvertWebp(name *string, data *[]byte) {
+	if b.GetBool("MediaConvertWebPToPNG") {
+		b.Log.Debugf("WebP to PNG conversion enabled, converting %v", name)
+		err := helper.ConvertWebPToPNG(data)
+		if err != nil {
+			b.Log.Errorf("conversion failed: %v", err)
+		} else {
+			*name = strings.Replace(*name, ".webp", ".png", 1)
+		}
+	}
+}
+
 // handleDownloadFile handles file download
 func (b *Btelegram) handleDownload(rmsg *config.Message, message *tgbotapi.Message) error {
 	size := 0
@@ -254,15 +304,18 @@ func (b *Btelegram) handleDownload(rmsg *config.Message, message *tgbotapi.Messa
 	if err != nil {
 		return err
 	}
-	if strings.HasSuffix(name, ".webp") && b.GetBool("MediaConvertWebPToPNG") {
+
-		b.Log.Debugf("WebP to PNG conversion enabled, converting %s", name)
+	if strings.HasSuffix(name, ".tgs.webp") {
-		err := helper.ConvertWebPToPNG(data)
+		b.maybeConvertTgs(&name, data)
-		if err != nil {
+	} else if strings.HasSuffix(name, ".webp") {
-			b.Log.Errorf("conversion failed: %s", err)
+		b.maybeConvertWebp(&name, data)
-		} else {
-			name = strings.Replace(name, ".webp", ".png", 1)
-		}
 	}
+
+	// rename .oga to .ogg  https://github.com/42wim/matterbridge/issues/906#issuecomment-741793512
+	if strings.HasSuffix(name, ".oga") && message.Audio != nil {
+		name = strings.Replace(name, ".oga", ".ogg", 1)
+	}
+
 	helper.HandleDownloadData(b.Log, rmsg, name, message.Caption, "", data, b.General)
 	return nil
 }
@@ -312,6 +365,9 @@ func (b *Btelegram) handleEdit(msg *config.Message, chatid int64) (string, error
 	case "Markdown":
 		b.Log.Debug("Using mode markdown")
 		m.ParseMode = tgbotapi.ModeMarkdown
+	case MarkdownV2:
+		b.Log.Debug("Using mode MarkdownV2")
+		m.ParseMode = MarkdownV2
 	}
 	if strings.ToLower(b.GetString("MessageFormat")) == HTMLNick {
 		b.Log.Debug("Using mode HTML - nick only")
@@ -333,7 +389,7 @@ func (b *Btelegram) handleUploadFile(msg *config.Message, chatid int64) string {
 			Name:  fi.Name,
 			Bytes: *fi.Data,
 		}
-		re := regexp.MustCompile(".(jpg|png)$")
+		re := regexp.MustCompile(".(jpg|jpe|png)$")
 		if re.MatchString(fi.Name) {
 			c = tgbotapi.NewPhotoUpload(chatid, file)
 		} else {

bridge/telegram/telegram.go

@@ -2,6 +2,7 @@ package btelegram
 
 import (
 	"html"
+	"log"
 	"strconv"
 	"strings"
 
@@ -15,6 +16,9 @@ const (
 	unknownUser = "unknown"
 	HTMLFormat  = "HTML"
 	HTMLNick    = "htmlnick"
+	MarkdownV2  = "MarkdownV2"
+	FormatPng   = "png"
+	FormatWebp  = "webp"
 )
 
 type Btelegram struct {
@@ -24,6 +28,16 @@ type Btelegram struct {
 }
 
 func New(cfg *bridge.Config) bridge.Bridger {
+	tgsConvertFormat := cfg.GetString("MediaConvertTgs")
+	if tgsConvertFormat != "" {
+		err := helper.CanConvertTgsToX()
+		if err != nil {
+			log.Fatalf("Telegram bridge configured to convert .tgs files to '%s', but lottie does not appear to work:\n%#v", tgsConvertFormat, err)
+		}
+		if tgsConvertFormat != FormatPng && tgsConvertFormat != FormatWebp {
+			log.Fatalf("Telegram bridge configured to convert .tgs files to '%s', but only '%s' and '%s' are supported.", FormatPng, FormatWebp, tgsConvertFormat)
+		}
+	}
 	return &Btelegram{Config: cfg, avatarMap: make(map[string]string)}
 }
 
@@ -126,6 +140,10 @@ func (b *Btelegram) sendMessage(chatid int64, username, text string) (string, er
 		b.Log.Debug("Using mode markdown")
 		m.ParseMode = tgbotapi.ModeMarkdown
 	}
+	if b.GetString("MessageFormat") == MarkdownV2 {
+		b.Log.Debug("Using mode MarkdownV2")
+		m.ParseMode = MarkdownV2
+	}
 	if strings.ToLower(b.GetString("MessageFormat")) == HTMLNick {
 		b.Log.Debug("Using mode HTML - nick only")
 		m.Text = username + html.EscapeString(text)

bridge/vk/vk.go

@@ -0,0 +1,327 @@
+package bvk
+
+import (
+	"bytes"
+	"context"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/42wim/matterbridge/bridge"
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/42wim/matterbridge/bridge/helper"
+
+	"github.com/SevereCloud/vksdk/v2/api"
+	"github.com/SevereCloud/vksdk/v2/events"
+	longpoll "github.com/SevereCloud/vksdk/v2/longpoll-bot"
+	"github.com/SevereCloud/vksdk/v2/object"
+)
+
+const (
+	audioMessage = "audio_message"
+	document     = "doc"
+	photo        = "photo"
+	video        = "video"
+	graffiti     = "graffiti"
+	sticker      = "sticker"
+	wall         = "wall"
+)
+
+type user struct {
+	lastname, firstname, avatar string
+}
+
+type Bvk struct {
+	c            *api.VK
+	usernamesMap map[int]user // cache of user names and avatar URLs
+	*bridge.Config
+}
+
+func New(cfg *bridge.Config) bridge.Bridger {
+	return &Bvk{usernamesMap: make(map[int]user), Config: cfg}
+}
+
+func (b *Bvk) Connect() error {
+	b.Log.Info("Connecting")
+	b.c = api.NewVK(b.GetString("Token"))
+	lp, err := longpoll.NewLongPoll(b.c, b.GetInt("GroupID"))
+	if err != nil {
+		b.Log.Debugf("%#v", err)
+
+		return err
+	}
+
+	lp.MessageNew(func(ctx context.Context, obj events.MessageNewObject) {
+		b.handleMessage(obj.Message, false)
+	})
+
+	b.Log.Info("Connection succeeded")
+
+	go func() {
+		err := lp.Run()
+		if err != nil {
+			b.Log.Fatal("Enable longpoll in group management")
+		}
+	}()
+
+	return nil
+}
+
+func (b *Bvk) Disconnect() error {
+	return nil
+}
+
+func (b *Bvk) JoinChannel(channel config.ChannelInfo) error {
+	return nil
+}
+
+func (b *Bvk) Send(msg config.Message) (string, error) {
+	b.Log.Debugf("=> Receiving %#v", msg)
+
+	peerID, err := strconv.Atoi(msg.Channel)
+	if err != nil {
+		return "", err
+	}
+
+	params := api.Params{}
+
+	text := msg.Username + msg.Text
+
+	if msg.Extra != nil {
+		if len(msg.Extra["file"]) > 0 {
+			// generate attachments string
+			attachment, urls := b.uploadFiles(msg.Extra, peerID)
+			params["attachment"] = attachment
+			text += urls
+		}
+	}
+
+	params["message"] = text
+
+	if msg.ID == "" {
+		// New message
+		params["random_id"] = time.Now().Unix()
+		params["peer_ids"] = msg.Channel
+
+		res, e := b.c.MessagesSendPeerIDs(params)
+		if e != nil {
+			return "", err
+		}
+
+		return strconv.Itoa(res[0].ConversationMessageID), nil
+	}
+	// Edit message
+	messageID, err := strconv.ParseInt(msg.ID, 10, 64)
+	if err != nil {
+		return "", err
+	}
+
+	params["peer_id"] = peerID
+	params["conversation_message_id"] = messageID
+
+	_, err = b.c.MessagesEdit(params)
+	if err != nil {
+		return "", err
+	}
+
+	return msg.ID, nil
+}
+
+func (b *Bvk) getUser(id int) user {
+	u, found := b.usernamesMap[id]
+	if !found {
+		b.Log.Debug("Fetching username for ", id)
+
+		if id >= 0 {
+			result, _ := b.c.UsersGet(api.Params{
+				"user_ids": id,
+				"fields":   "photo_200",
+			})
+
+			resUser := result[0]
+			u = user{lastname: resUser.LastName, firstname: resUser.FirstName, avatar: resUser.Photo200}
+			b.usernamesMap[id] = u
+		} else {
+			result, _ := b.c.GroupsGetByID(api.Params{
+				"group_id": id * -1,
+			})
+
+			resGroup := result[0]
+			u = user{lastname: resGroup.Name, avatar: resGroup.Photo200}
+		}
+	}
+
+	return u
+}
+
+func (b *Bvk) handleMessage(msg object.MessagesMessage, isFwd bool) {
+	b.Log.Debug("ChatID: ", msg.PeerID)
+	// fetch user info
+	u := b.getUser(msg.FromID)
+
+	rmsg := config.Message{
+		Text:     msg.Text,
+		Username: u.firstname + " " + u.lastname,
+		Avatar:   u.avatar,
+		Channel:  strconv.Itoa(msg.PeerID),
+		Account:  b.Account,
+		UserID:   strconv.Itoa(msg.FromID),
+		ID:       strconv.Itoa(msg.ConversationMessageID),
+		Extra:    make(map[string][]interface{}),
+	}
+
+	if msg.ReplyMessage != nil {
+		ur := b.getUser(msg.ReplyMessage.FromID)
+		rmsg.Text = "Re: " + ur.firstname + " " + ur.lastname + "\n" + rmsg.Text
+	}
+
+	if isFwd {
+		rmsg.Username = "Fwd: " + rmsg.Username
+	}
+
+	if len(msg.Attachments) > 0 {
+		urls, text := b.getFiles(msg.Attachments)
+
+		if text != "" {
+			rmsg.Text += "\n" + text
+		}
+
+		// download
+		b.downloadFiles(&rmsg, urls)
+	}
+
+	if len(msg.FwdMessages) > 0 {
+		rmsg.Text += strconv.Itoa(len(msg.FwdMessages)) + " forwarded messages"
+	}
+
+	b.Remote <- rmsg
+
+	if len(msg.FwdMessages) > 0 {
+		// recursive processing of forwarded messages
+		for _, m := range msg.FwdMessages {
+			m.PeerID = msg.PeerID
+			b.handleMessage(m, true)
+		}
+	}
+}
+
+func (b *Bvk) uploadFiles(extra map[string][]interface{}, peerID int) (string, string) {
+	var attachments []string
+	text := ""
+
+	for _, f := range extra["file"] {
+		fi := f.(config.FileInfo)
+
+		if fi.Comment != "" {
+			text += fi.Comment + "\n"
+		}
+		a, err := b.uploadFile(fi, peerID)
+		if err != nil {
+			b.Log.Error("File upload error ", fi.Name)
+		}
+
+		attachments = append(attachments, a)
+	}
+
+	return strings.Join(attachments, ","), text
+}
+
+func (b *Bvk) uploadFile(file config.FileInfo, peerID int) (string, error) {
+	r := bytes.NewReader(*file.Data)
+
+	photoRE := regexp.MustCompile(".(jpg|jpe|png)$")
+	if photoRE.MatchString(file.Name) {
+		p, err := b.c.UploadMessagesPhoto(peerID, r)
+		if err != nil {
+			return "", err
+		}
+
+		return photo + strconv.Itoa(p[0].OwnerID) + "_" + strconv.Itoa(p[0].ID), nil
+	}
+
+	var doctype string
+	if strings.Contains(file.Name, ".ogg") {
+		doctype = audioMessage
+	} else {
+		doctype = document
+	}
+
+	doc, err := b.c.UploadMessagesDoc(peerID, doctype, file.Name, "", r)
+	if err != nil {
+		return "", err
+	}
+
+	switch doc.Type {
+	case audioMessage:
+		return document + strconv.Itoa(doc.AudioMessage.OwnerID) + "_" + strconv.Itoa(doc.AudioMessage.ID), nil
+	case document:
+		return document + strconv.Itoa(doc.Doc.OwnerID) + "_" + strconv.Itoa(doc.Doc.ID), nil
+	}
+
+	return "", nil
+}
+
+func (b *Bvk) getFiles(attachments []object.MessagesMessageAttachment) ([]string, string) {
+	var urls []string
+	var text []string
+
+	for _, a := range attachments {
+		switch a.Type {
+		case photo:
+			var resolution float64 = 0
+			url := a.Photo.Sizes[0].URL
+			for _, size := range a.Photo.Sizes {
+				r := size.Height * size.Width
+				if resolution < r {
+					resolution = r
+					url = size.URL
+				}
+			}
+
+			urls = append(urls, url)
+
+		case document:
+			urls = append(urls, a.Doc.URL)
+
+		case graffiti:
+			urls = append(urls, a.Graffiti.URL)
+
+		case audioMessage:
+			urls = append(urls, a.AudioMessage.DocsDocPreviewAudioMessage.LinkOgg)
+
+		case sticker:
+			var resolution float64 = 0
+			url := a.Sticker.Images[0].URL
+			for _, size := range a.Sticker.Images {
+				r := size.Height * size.Width
+				if resolution < r {
+					resolution = r
+					url = size.URL
+				}
+			}
+			urls = append(urls, url+".png")
+		case video:
+			text = append(text, "https://vk.com/video"+strconv.Itoa(a.Video.OwnerID)+"_"+strconv.Itoa(a.Video.ID))
+
+		case wall:
+			text = append(text, "https://vk.com/wall"+strconv.Itoa(a.Wall.FromID)+"_"+strconv.Itoa(a.Wall.ID))
+
+		default:
+			text = append(text, "This attachment is not supported ("+a.Type+")")
+		}
+	}
+
+	return urls, strings.Join(text, "\n")
+}
+
+func (b *Bvk) downloadFiles(rmsg *config.Message, urls []string) {
+	for _, url := range urls {
+		data, err := helper.DownloadFile(url)
+		if err == nil {
+			urlPart := strings.Split(url, "/")
+			name := strings.Split(urlPart[len(urlPart)-1], "?")[0]
+			helper.HandleDownloadData(b.Log, rmsg, name, "", url, data, b.General)
+		}
+	}
+}

bridge/whatsapp/handlers.go

@@ -9,6 +9,7 @@ import (
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/42wim/matterbridge/bridge/helper"
 	"github.com/Rhymen/go-whatsapp"
+	"github.com/jpillora/backoff"
 )
 
 /*
@@ -22,15 +23,55 @@ Check:
 // HandleError received from WhatsApp
 func (b *Bwhatsapp) HandleError(err error) {
 	// ignore received invalid data errors. https://github.com/42wim/matterbridge/issues/843
-	if strings.Contains(err.Error(), "error processing data: received invalid data") {
+	// ignore tag 174 errors. https://github.com/42wim/matterbridge/issues/1094
+	if strings.Contains(err.Error(), "error processing data: received invalid data") ||
+		strings.Contains(err.Error(), "invalid string with tag 174") {
 		return
 	}
-	b.Log.Errorf("%v", err) // TODO implement proper handling? at least respond to different error types
+
+	switch err.(type) {
+	case *whatsapp.ErrConnectionClosed, *whatsapp.ErrConnectionFailed:
+		b.reconnect(err)
+	default:
+		switch err {
+		case whatsapp.ErrConnectionTimeout:
+			b.reconnect(err)
+		default:
+			b.Log.Errorf("%v", err)
+		}
+	}
+}
+
+func (b *Bwhatsapp) reconnect(err error) {
+	bf := &backoff.Backoff{
+		Min:    time.Second,
+		Max:    5 * time.Minute,
+		Jitter: true,
+	}
+
+	for {
+		d := bf.Duration()
+
+		b.Log.Errorf("Connection failed, underlying error: %v", err)
+		b.Log.Infof("Waiting %s...", d)
+
+		time.Sleep(d)
+
+		b.Log.Info("Reconnecting...")
+
+		err := b.conn.Restore()
+		if err == nil {
+			bf.Reset()
+			b.startedAt = uint64(time.Now().Unix())
+
+			return
+		}
+	}
 }
 
 // HandleTextMessage sent from WhatsApp, relay it to the brige
 func (b *Bwhatsapp) HandleTextMessage(message whatsapp.TextMessage) {
-	if message.Info.FromMe { // || !strings.Contains(strings.ToLower(message.Text), "@echo") {
+	if message.Info.FromMe {
 		return
 	}
 	// whatsapp sends last messages to show context , cut them
@@ -38,13 +79,13 @@ func (b *Bwhatsapp) HandleTextMessage(message whatsapp.TextMessage) {
 		return
 	}
 
-	messageTime := time.Unix(int64(message.Info.Timestamp), 0) // TODO check how behaves between timezones
 	groupJID := message.Info.RemoteJid
-
 	senderJID := message.Info.SenderJid
+
 	if len(senderJID) == 0 {
-		// TODO workaround till https://github.com/Rhymen/go-whatsapp/issues/86 resolved
+		if message.Info.Source != nil && message.Info.Source.Participant != nil {
-		senderJID = *message.Info.Source.Participant
+			senderJID = *message.Info.Source.Participant
+		}
 	}
 
 	// translate sender's JID to the nicest username we can get
@@ -65,108 +106,205 @@ func (b *Bwhatsapp) HandleTextMessage(message whatsapp.TextMessage) {
 			if mention == "" {
 				mention = "someone"
 			}
+
 			message.Text = strings.Replace(message.Text, "@"+numberAndSuffix[0], "@"+mention, 1)
 		}
 	}
 
-	b.Log.Debugf("<= Sending message from %s on %s to gateway", senderJID, b.Account)
 	rmsg := config.Message{
-		UserID:    senderJID,
+		UserID:   senderJID,
-		Username:  senderName,
+		Username: senderName,
-		Text:      message.Text,
+		Text:     message.Text,
-		Timestamp: messageTime,
+		Channel:  groupJID,
-		Channel:   groupJID,
+		Account:  b.Account,
-		Account:   b.Account,
+		Protocol: b.Protocol,
-		Protocol:  b.Protocol,
+		Extra:    make(map[string][]interface{}),
-		Extra:     make(map[string][]interface{}),
 		//	ParentID: TODO, // TODO handle thread replies  // map from Info.QuotedMessageID string
-		//	Event     string    `json:"event"`
+		ID: message.Info.Id,
-		//	Gateway   string  // will be added during message processing
+	}
-		ID: message.Info.Id}
 
 	if avatarURL, exists := b.userAvatars[senderJID]; exists {
 		rmsg.Avatar = avatarURL
 	}
 
+	b.Log.Debugf("<= Sending message from %s on %s to gateway", senderJID, b.Account)
 	b.Log.Debugf("<= Message is %#v", rmsg)
+
 	b.Remote <- rmsg
 }
 
 // HandleImageMessage sent from WhatsApp, relay it to the brige
 func (b *Bwhatsapp) HandleImageMessage(message whatsapp.ImageMessage) {
-	if message.Info.FromMe { // || !strings.Contains(strings.ToLower(message.Text), "@echo") {
+	if message.Info.FromMe || message.Info.Timestamp < b.startedAt {
 		return
 	}
 
-	// whatsapp sends last messages to show context , cut them
-	if message.Info.Timestamp < b.startedAt {
-		return
-	}
-
-	messageTime := time.Unix(int64(message.Info.Timestamp), 0) // TODO check how behaves between timezones
-	groupJID := message.Info.RemoteJid
-
 	senderJID := message.Info.SenderJid
-	// if len(senderJid) == 0 {
+	if len(message.Info.SenderJid) == 0 && message.Info.Source != nil && message.Info.Source.Participant != nil {
-	//   // TODO workaround till https://github.com/Rhymen/go-whatsapp/issues/86 resolved
+		senderJID = *message.Info.Source.Participant
-	//   senderJid = *message.Info.Source.Participant
+	}
-	// }
 
-	// translate sender's Jid to the nicest username we can get
+	senderName := b.getSenderName(message.Info.SenderJid)
-	senderName := b.getSenderName(senderJID)
 	if senderName == "" {
 		senderName = "Someone" // don't expose telephone number
 	}
 
-	b.Log.Debugf("<= Sending message from %s on %s to gateway", senderJID, b.Account)
 	rmsg := config.Message{
-		UserID:    senderJID,
+		UserID:   senderJID,
-		Username:  senderName,
+		Username: senderName,
-		Timestamp: messageTime,
+		Channel:  message.Info.RemoteJid,
-		Channel:   groupJID,
+		Account:  b.Account,
-		Account:   b.Account,
+		Protocol: b.Protocol,
-		Protocol:  b.Protocol,
+		Extra:    make(map[string][]interface{}),
-		Extra:     make(map[string][]interface{}),
+		ID:       message.Info.Id,
-		//  ParentID: TODO,      // TODO handle thread replies  // map from Info.QuotedMessageID string
+	}
-		//  Event     string    `json:"event"`
-		//  Gateway   string     // will be added during message processing
-		ID: message.Info.Id}
 
 	if avatarURL, exists := b.userAvatars[senderJID]; exists {
 		rmsg.Avatar = avatarURL
 	}
 
-	// Download and unencrypt content
+	fileExt, err := mime.ExtensionsByType(message.Type)
-	data, err := message.Download()
 	if err != nil {
-		b.Log.Errorf("%v", err)
+		b.Log.Errorf("Mimetype detection error: %s", err)
+
 		return
 	}
 
-	// Get file extension by mimetype
+	// rename .jfif to .jpg https://github.com/42wim/matterbridge/issues/1292
+	if fileExt[0] == ".jfif" {
+		fileExt[0] = ".jpg"
+	}
+
+	filename := fmt.Sprintf("%v%v", message.Info.Id, fileExt[0])
+
+	b.Log.Debugf("Trying to download %s with type %s", filename, message.Type)
+
+	data, err := message.Download()
+	if err != nil {
+		b.Log.Errorf("Download image failed: %s", err)
+
+		return
+	}
+
+	// Move file to bridge storage
+	helper.HandleDownloadData(b.Log, &rmsg, filename, message.Caption, "", &data, b.General)
+
+	b.Log.Debugf("<= Sending message from %s on %s to gateway", senderJID, b.Account)
+	b.Log.Debugf("<= Message is %#v", rmsg)
+
+	b.Remote <- rmsg
+}
+
+// HandleVideoMessage downloads video messages
+func (b *Bwhatsapp) HandleVideoMessage(message whatsapp.VideoMessage) {
+	if message.Info.FromMe || message.Info.Timestamp < b.startedAt {
+		return
+	}
+
+	senderJID := message.Info.SenderJid
+	if len(message.Info.SenderJid) == 0 && message.Info.Source != nil && message.Info.Source.Participant != nil {
+		senderJID = *message.Info.Source.Participant
+	}
+
+	senderName := b.getSenderName(message.Info.SenderJid)
+	if senderName == "" {
+		senderName = "Someone" // don't expose telephone number
+	}
+
+	rmsg := config.Message{
+		UserID:   senderJID,
+		Username: senderName,
+		Channel:  message.Info.RemoteJid,
+		Account:  b.Account,
+		Protocol: b.Protocol,
+		Extra:    make(map[string][]interface{}),
+		ID:       message.Info.Id,
+	}
+
+	if avatarURL, exists := b.userAvatars[senderJID]; exists {
+		rmsg.Avatar = avatarURL
+	}
+
 	fileExt, err := mime.ExtensionsByType(message.Type)
 	if err != nil {
-		b.Log.Errorf("%v", err)
+		b.Log.Errorf("Mimetype detection error: %s", err)
+
 		return
 	}
 
 	filename := fmt.Sprintf("%v%v", message.Info.Id, fileExt[0])
 
-	b.Log.Debugf("<= Image downloaded and unencrypted")
+	b.Log.Debugf("Trying to download %s with size %#v and type %s", filename, message.Length, message.Type)
+
+	data, err := message.Download()
+	if err != nil {
+		b.Log.Errorf("Download video failed: %s", err)
+
+		return
+	}
 
 	// Move file to bridge storage
 	helper.HandleDownloadData(b.Log, &rmsg, filename, message.Caption, "", &data, b.General)
 
-	b.Log.Debugf("<= Image Message is %#v", rmsg)
+	b.Log.Debugf("<= Sending message from %s on %s to gateway", senderJID, b.Account)
+	b.Log.Debugf("<= Message is %#v", rmsg)
+
 	b.Remote <- rmsg
 }
 
-//func (b *Bwhatsapp) HandleVideoMessage(message whatsapp.VideoMessage) {
+// HandleAudioMessage downloads audio messages
-//	fmt.Println(message) // TODO implement
+func (b *Bwhatsapp) HandleAudioMessage(message whatsapp.AudioMessage) {
-//}
+	if message.Info.FromMe || message.Info.Timestamp < b.startedAt {
-//
+		return
-//func (b *Bwhatsapp) HandleJsonMessage(message string) {
+	}
-//	fmt.Println(message) // TODO implement
+
-//}
+	senderJID := message.Info.SenderJid
-// TODO HandleRawMessage
+	if len(message.Info.SenderJid) == 0 && message.Info.Source != nil && message.Info.Source.Participant != nil {
-// TODO HandleAudioMessage
+		senderJID = *message.Info.Source.Participant
+	}
+
+	senderName := b.getSenderName(message.Info.SenderJid)
+	if senderName == "" {
+		senderName = "Someone" // don't expose telephone number
+	}
+
+	rmsg := config.Message{
+		UserID:   senderJID,
+		Username: senderName,
+		Channel:  message.Info.RemoteJid,
+		Account:  b.Account,
+		Protocol: b.Protocol,
+		Extra:    make(map[string][]interface{}),
+		ID:       message.Info.Id,
+	}
+
+	if avatarURL, exists := b.userAvatars[senderJID]; exists {
+		rmsg.Avatar = avatarURL
+	}
+
+	fileExt, err := mime.ExtensionsByType(message.Type)
+	if err != nil {
+		b.Log.Errorf("Mimetype detection error: %s", err)
+
+		return
+	}
+
+	filename := fmt.Sprintf("%v%v", message.Info.Id, fileExt[0])
+
+	b.Log.Debugf("Trying to download %s with size %#v and type %s", filename, message.Length, message.Type)
+
+	data, err := message.Download()
+	if err != nil {
+		b.Log.Errorf("Download audio failed: %s", err)
+
+		return
+	}
+
+	// Move file to bridge storage
+	helper.HandleDownloadData(b.Log, &rmsg, filename, "audio message", "", &data, b.General)
+
+	b.Log.Debugf("<= Sending message from %s on %s to gateway", senderJID, b.Account)
+	b.Log.Debugf("<= Message is %#v", rmsg)
+
+	b.Remote <- rmsg
+}

bridge/whatsapp/helpers.go

@@ -6,22 +6,24 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"strings"
 
 	qrcodeTerminal "github.com/Baozisoftware/qrcode-terminal-go"
 	"github.com/Rhymen/go-whatsapp"
 )
 
 type ProfilePicInfo struct {
-	URL string `json:"eurl"`
+	URL    string `json:"eurl"`
-	Tag string `json:"tag"`
+	Tag    string `json:"tag"`
-
+	Status int16  `json:"status"`
-	Status int16 `json:"status"`
 }
 
 func qrFromTerminal(invert bool) chan string {
 	qr := make(chan string)
+
 	go func() {
 		terminal := qrcodeTerminal.New()
+
 		if invert {
 			terminal = qrcodeTerminal.New2(qrcodeTerminal.ConsoleColors.BrightWhite, qrcodeTerminal.ConsoleColors.BrightBlack, qrcodeTerminal.QRCodeRecoveryLevels.Medium)
 		}
@@ -44,13 +46,12 @@ func (b *Bwhatsapp) readSession() (whatsapp.Session, error) {
 	if err != nil {
 		return session, err
 	}
+
 	defer file.Close()
+
 	decoder := gob.NewDecoder(file)
-	err = decoder.Decode(&session)
+
-	if err != nil {
+	return session, decoder.Decode(&session)
-		return session, err
-	}
-	return session, nil
 }
 
 func (b *Bwhatsapp) writeSession(session whatsapp.Session) error {
@@ -65,11 +66,31 @@ func (b *Bwhatsapp) writeSession(session whatsapp.Session) error {
 	if err != nil {
 		return err
 	}
-	defer file.Close()
-	encoder := gob.NewEncoder(file)
-	err = encoder.Encode(session)
 
-	return err
+	defer file.Close()
+
+	encoder := gob.NewEncoder(file)
+
+	return encoder.Encode(session)
+}
+
+func (b *Bwhatsapp) restoreSession() (*whatsapp.Session, error) {
+	session, err := b.readSession()
+	if err != nil {
+		b.Log.Warn(err.Error())
+	}
+
+	b.Log.Debugln("Restoring WhatsApp session..")
+
+	session, err = b.conn.RestoreWithSession(session)
+	if err != nil {
+		// restore session connection timed out (I couldn't get over it without logging in again)
+		return nil, errors.New("failed to restore session: " + err.Error())
+	}
+
+	b.Log.Debugln("Session restored successfully!")
+
+	return &session, nil
 }
 
 func (b *Bwhatsapp) getSenderName(senderJid string) string {
@@ -80,8 +101,33 @@ func (b *Bwhatsapp) getSenderName(senderJid string) string {
 		// if user is not in phone contacts
 		// it is the most obvious scenario unless you sync your phone contacts with some remote updated source
 		// users can change it in their WhatsApp settings -> profile -> click on Avatar
-		return sender.Notify
+		if sender.Notify != "" {
+			return sender.Notify
+		}
+
+		if sender.Short != "" {
+			return sender.Short
+		}
 	}
+
+	// try to reload this contact
+	_, err := b.conn.Contacts()
+	if err != nil {
+		b.Log.Errorf("error on update of contacts: %v", err)
+	}
+
+	if contact, exists := b.conn.Store.Contacts[senderJid]; exists {
+		// Add it to the user map
+		b.users[senderJid] = contact
+
+		if contact.Name != "" {
+			return contact.Name
+		}
+		// if user is not in phone contacts
+		// same as above
+		return contact.Notify
+	}
+
 	return ""
 }
 
@@ -89,6 +135,7 @@ func (b *Bwhatsapp) getSenderNotify(senderJid string) string {
 	if sender, exists := b.users[senderJid]; exists {
 		return sender.Notify
 	}
+
 	return ""
 }
 
@@ -97,11 +144,20 @@ func (b *Bwhatsapp) GetProfilePicThumb(jid string) (*ProfilePicInfo, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to get avatar: %v", err)
 	}
+
 	content := <-data
 	info := &ProfilePicInfo{}
+
 	err = json.Unmarshal([]byte(content), info)
 	if err != nil {
 		return info, fmt.Errorf("failed to unmarshal avatar info: %v", err)
 	}
+
 	return info, nil
 }
+
+func isGroupJid(identifier string) bool {
+	return strings.HasSuffix(identifier, "@g.us") ||
+		strings.HasSuffix(identifier, "@temp") ||
+		strings.HasSuffix(identifier, "@broadcast")
+}

bridge/whatsapp/whatsapp.go

@@ -28,7 +28,6 @@ const (
 type Bwhatsapp struct {
 	*bridge.Config
 
-	// https://github.com/Rhymen/go-whatsapp/blob/c31092027237441cffba1b9cb148eadf7c83c3d2/session.go#L18-L21
 	session   *whatsapp.Session
 	conn      *whatsapp.Conn
 	startedAt uint64
@@ -40,6 +39,7 @@ type Bwhatsapp struct {
 // New Create a new WhatsApp bridge. This will be called for each [whatsapp.<server>] entry you have in the config file
 func New(cfg *bridge.Config) bridge.Bridger {
 	number := cfg.GetString(cfgNumber)
+
 	if number == "" {
 		cfg.Log.Fatalf("Missing configuration for WhatsApp bridge: Number")
 	}
@@ -50,21 +50,17 @@ func New(cfg *bridge.Config) bridge.Bridger {
 		users:       make(map[string]whatsapp.Contact),
 		userAvatars: make(map[string]string),
 	}
+
 	return b
 }
 
 // Connect to WhatsApp. Required implementation of the Bridger interface
-// https://github.com/42wim/matterbridge/blob/2cfd880cdb0df29771bf8f31df8d990ab897889d/bridge/bridge.go#L11-L16
 func (b *Bwhatsapp) Connect() error {
-	b.RLock() // TODO do we need locking for Whatsapp?
-	defer b.RUnlock()
-
 	number := b.GetString(cfgNumber)
 	if number == "" {
-		return errors.New("WhatsApp's telephone Number need to be configured")
+		return errors.New("whatsapp's telephone number need to be configured")
 	}
 
-	// https://github.com/Rhymen/go-whatsapp#creating-a-connection
 	b.Log.Debugln("Connecting to WhatsApp..")
 	conn, err := whatsapp.NewConn(20 * time.Second)
 	if err != nil {
@@ -77,35 +73,18 @@ func (b *Bwhatsapp) Connect() error {
 	b.Log.Debugln("WhatsApp connection successful")
 
 	// load existing session in order to keep it between restarts
-	if b.session == nil {
+	b.session, err = b.restoreSession()
-		var session whatsapp.Session
+	if err != nil {
-		session, err = b.readSession()
+		b.Log.Warn(err.Error())
-
-		if err == nil {
-			b.Log.Debugln("Restoring WhatsApp session..")
-
-			// https://github.com/Rhymen/go-whatsapp#restore
-			session, err = b.conn.RestoreWithSession(session)
-			if err != nil {
-				// TODO return or continue to normal login?
-				// restore session connection timed out (I couldn't get over it without logging in again)
-				return errors.New("failed to restore session: " + err.Error())
-			}
-
-			b.session = &session
-			b.Log.Debugln("Session restored successfully!")
-		} else {
-			b.Log.Warn(err.Error())
-		}
 	}
 
 	// login to a new session
 	if b.session == nil {
-		err = b.Login()
+		if err = b.Login(); err != nil {
-		if err != nil {
 			return err
 		}
 	}
+
 	b.startedAt = uint64(time.Now().Unix())
 
 	_, err = b.conn.Contacts()
@@ -113,6 +92,13 @@ func (b *Bwhatsapp) Connect() error {
 		return fmt.Errorf("error on update of contacts: %v", err)
 	}
 
+	// see https://github.com/Rhymen/go-whatsapp/issues/137#issuecomment-480316013
+	for len(b.conn.Store.Contacts) == 0 {
+		b.conn.Contacts() // nolint:errcheck
+
+		<-time.After(1 * time.Second)
+	}
+
 	// map all the users
 	for id, contact := range b.conn.Store.Contacts {
 		if !isGroupJid(id) && id != "status@broadcast" {
@@ -129,12 +115,13 @@ func (b *Bwhatsapp) Connect() error {
 			info, err := b.GetProfilePicThumb(jid)
 			if err != nil {
 				b.Log.Warnf("Could not get profile photo of %s: %v", jid, err)
-
 			} else {
-				// TODO any race conditions here?
+				b.Lock()
 				b.userAvatars[jid] = info.URL
+				b.Unlock()
 			}
 		}
+
 		b.Log.Debug("Finished getting avatars..")
 	}()
 
@@ -151,8 +138,10 @@ func (b *Bwhatsapp) Login() error {
 	session, err := b.conn.Login(qrChan)
 	if err != nil {
 		b.Log.Warnln("Failed to log in:", err)
+
 		return err
 	}
+
 	b.session = &session
 
 	b.Log.Infof("Logged into session: %#v", session)
@@ -163,74 +152,62 @@ func (b *Bwhatsapp) Login() error {
 		fmt.Fprintf(os.Stderr, "error saving session: %v\n", err)
 	}
 
-	// TODO change connection strings to configured ones longClientName:"github.com/rhymen/go-whatsapp", shortClientName:"go-whatsapp"}" prefix=whatsapp
-	// TODO get also a nice logo
-
-	// TODO notification about unplugged and dead battery
-	// conn.Info: Wid, Pushname, Connected, Battery, Plugged
-
 	return nil
 }
 
 // Disconnect is called while reconnecting to the bridge
-// TODO 42wim Documentation would be helpful on when reconnects happen and what should be done in this function
 // Required implementation of the Bridger interface
-// https://github.com/42wim/matterbridge/blob/2cfd880cdb0df29771bf8f31df8d990ab897889d/bridge/bridge.go#L11-L16
 func (b *Bwhatsapp) Disconnect() error {
 	// We could Logout, but that would close the session completely and would require a new QR code scan
 	// https://github.com/Rhymen/go-whatsapp/blob/c31092027237441cffba1b9cb148eadf7c83c3d2/session.go#L377-L381
 	return nil
 }
 
-func isGroupJid(identifier string) bool {
-	return strings.HasSuffix(identifier, "@g.us") || strings.HasSuffix(identifier, "@temp")
-}
-
 // JoinChannel Join a WhatsApp group specified in gateway config as channel='number-id@g.us' or channel='Channel name'
 // Required implementation of the Bridger interface
 // https://github.com/42wim/matterbridge/blob/2cfd880cdb0df29771bf8f31df8d990ab897889d/bridge/bridge.go#L11-L16
 func (b *Bwhatsapp) JoinChannel(channel config.ChannelInfo) error {
 	byJid := isGroupJid(channel.Name)
 
+	// see https://github.com/Rhymen/go-whatsapp/issues/137#issuecomment-480316013
+	for len(b.conn.Store.Contacts) == 0 {
+		b.conn.Contacts() // nolint:errcheck
+		<-time.After(1 * time.Second)
+	}
+
 	// verify if we are member of the given group
 	if byJid {
 		// channel.Name specifies static group jID, not the name
 		if _, exists := b.conn.Store.Contacts[channel.Name]; !exists {
 			return fmt.Errorf("account doesn't belong to group with jid %s", channel.Name)
 		}
-	} else {
-		// channel.Name specifies group name that might change, warn about it
-		var jids []string
-		for id, contact := range b.conn.Store.Contacts {
-			if isGroupJid(id) && contact.Name == channel.Name {
-				jids = append(jids, id)
-			}
-		}
 
-		switch len(jids) {
+		return nil
-		case 0:
+	}
-			// didn't match any group - print out possibilites
-			// TODO sort
-			// copy b;
-			//sort.Slice(people, func(i, j int) bool {
-			//	return people[i].Age > people[j].Age
-			//})
-			for id, contact := range b.conn.Store.Contacts {
-				if isGroupJid(id) {
-					b.Log.Infof("%s %s", contact.Jid, contact.Name)
-				}
-			}
-			return fmt.Errorf("please specify group's JID from the list above instead of the name '%s'", channel.Name)
 
-		case 1:
+	// channel.Name specifies group name that might change, warn about it
-			return fmt.Errorf("group name might change. Please configure gateway with channel=\"%v\" instead of channel=\"%v\"", jids[0], channel.Name)
+	var jids []string
-
+	for id, contact := range b.conn.Store.Contacts {
-		default:
+		if isGroupJid(id) && contact.Name == channel.Name {
-			return fmt.Errorf("there is more than one group with name '%s'. Please specify one of JIDs as channel name: %v", channel.Name, jids)
+			jids = append(jids, id)
 		}
 	}
 
-	return nil
+	switch len(jids) {
+	case 0:
+		// didn't match any group - print out possibilites
+		for id, contact := range b.conn.Store.Contacts {
+			if isGroupJid(id) {
+				b.Log.Infof("%s %s", contact.Jid, contact.Name)
+			}
+		}
+
+		return fmt.Errorf("please specify group's JID from the list above instead of the name '%s'", channel.Name)
+	case 1:
+		return fmt.Errorf("group name might change. Please configure gateway with channel=\"%v\" instead of channel=\"%v\"", jids[0], channel.Name)
+	default:
+		return fmt.Errorf("there is more than one group with name '%s'. Please specify one of JIDs as channel name: %v", channel.Name, jids)
+	}
 }
 
 // Post a document message from the bridge to WhatsApp
@@ -304,14 +281,12 @@ func (b *Bwhatsapp) Send(msg config.Message) (string, error) {
 		if msg.ID == "" {
 			// No message ID in case action is executed on a message sent before the bridge was started
 			// and then the bridge cache doesn't have this message ID mapped
-
-			// TODO 42wim Doesn't the app get clogged with a ton of IDs after some time of running?
-			// WhatsApp allows to set any ID so in that case we could use external IDs and don't do mapping
-			// but external IDs are not set
 			return "", nil
 		}
-		// TODO delete message on WhatsApp https://github.com/Rhymen/go-whatsapp/issues/100
+
-		return "", nil
+		_, err := b.conn.RevokeMessage(msg.Channel, msg.ID, true)
+
+		return "", err
 	}
 
 	// Edit message
@@ -319,7 +294,6 @@ func (b *Bwhatsapp) Send(msg config.Message) (string, error) {
 		b.Log.Debugf("updating message with id %s", msg.ID)
 
 		msg.Text += " (edited)"
-		// TODO handle edit as a message reply with updated text
 	}
 
 	// Handle Upload a file
@@ -349,16 +323,7 @@ func (b *Bwhatsapp) Send(msg config.Message) (string, error) {
 
 	b.Log.Debugf("=> Sending %#v", msg)
 
-	// create message ID
+	return b.conn.Send(message)
-	// TODO follow and act if https://github.com/Rhymen/go-whatsapp/issues/101 implemented
-	idBytes := make([]byte, 10)
-	if _, err := rand.Read(idBytes); err != nil {
-		b.Log.Warn(err.Error())
-	}
-	message.Info.Id = strings.ToUpper(hex.EncodeToString(idBytes))
-	_, err := b.conn.Send(message)
-
-	return message.Info.Id, err
 }
 
 // TODO do we want that? to allow login with QR code from a bridged channel? https://github.com/tulir/mautrix-whatsapp/blob/513eb18e2d59bada0dd515ee1abaaf38a3bfe3d5/commands.go#L76

bridge/xmpp/handler.go

@@ -0,0 +1,34 @@
+package bxmpp
+
+import (
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/42wim/matterbridge/bridge/helper"
+	"github.com/matterbridge/go-xmpp"
+)
+
+// handleDownloadAvatar downloads the avatar of userid from channel
+// sends a EVENT_AVATAR_DOWNLOAD message to the gateway if successful.
+// logs an error message if it fails
+func (b *Bxmpp) handleDownloadAvatar(avatar xmpp.AvatarData) {
+	rmsg := config.Message{
+		Username: "system",
+		Text:     "avatar",
+		Channel:  b.parseChannel(avatar.From),
+		Account:  b.Account,
+		UserID:   avatar.From,
+		Event:    config.EventAvatarDownload,
+		Extra:    make(map[string][]interface{}),
+	}
+	if _, ok := b.avatarMap[avatar.From]; !ok {
+		b.Log.Debugf("Avatar.From: %s", avatar.From)
+
+		err := helper.HandleDownloadSize(b.Log, &rmsg, avatar.From+".png", int64(len(avatar.Data)), b.General)
+		if err != nil {
+			b.Log.Error(err)
+			return
+		}
+		helper.HandleDownloadData(b.Log, &rmsg, avatar.From+".png", rmsg.Text, "", &avatar.Data, b.General)
+		b.Log.Debugf("Avatar download complete")
+		b.Remote <- rmsg
+	}
+}

bridge/xmpp/helpers.go

@@ -0,0 +1,30 @@
+package bxmpp
+
+import (
+	"regexp"
+
+	"github.com/42wim/matterbridge/bridge/config"
+)
+
+var pathRegex = regexp.MustCompile("[^a-zA-Z0-9]+")
+
+// GetAvatar constructs a URL for a given user-avatar if it is available in the cache.
+func getAvatar(av map[string]string, userid string, general *config.Protocol) string {
+	if hash, ok := av[userid]; ok {
+		// NOTE: This does not happen in bridge/helper/helper.go but messes up XMPP
+		id := pathRegex.ReplaceAllString(userid, "_")
+		return general.MediaServerDownload + "/" + hash + "/" + id + ".png"
+	}
+	return ""
+}
+
+func (b *Bxmpp) cacheAvatar(msg *config.Message) string {
+	fi := msg.Extra["file"][0].(config.FileInfo)
+	/* if we have a sha we have successfully uploaded the file to the media server,
+	so we can now cache the sha */
+	if fi.SHA != "" {
+		b.Log.Debugf("Added %s to %s in avatarMap", fi.SHA, msg.UserID)
+		b.avatarMap[msg.UserID] = fi.SHA
+	}
+	return ""
+}

bridge/xmpp/xmpp.go

@@ -1,8 +1,11 @@
 package bxmpp
 
 import (
+	"bytes"
 	"crypto/tls"
+	"encoding/json"
 	"fmt"
+	"net/http"
 	"strings"
 	"sync"
 	"time"
@@ -23,12 +26,17 @@ type Bxmpp struct {
 	xmppMap   map[string]string
 	connected bool
 	sync.RWMutex
+
+	avatarAvailability map[string]bool
+	avatarMap          map[string]string
 }
 
 func New(cfg *bridge.Config) bridge.Bridger {
 	return &Bxmpp{
-		Config:  cfg,
+		Config:             cfg,
-		xmppMap: make(map[string]string),
+		xmppMap:            make(map[string]string),
+		avatarAvailability: make(map[string]bool),
+		avatarMap:          make(map[string]string),
 	}
 }
 
@@ -67,17 +75,35 @@ func (b *Bxmpp) Send(msg config.Message) (string, error) {
 	if msg.Event == config.EventMsgDelete {
 		return "", nil
 	}
+
 	b.Log.Debugf("=> Receiving %#v", msg)
 
+	if msg.Event == config.EventAvatarDownload {
+		return b.cacheAvatar(&msg), nil
+	}
+
+	// Make a action /me of the message, prepend the username with it.
+	// https://xmpp.org/extensions/xep-0245.html
+	if msg.Event == config.EventUserAction {
+		msg.Username = "/me " + msg.Username
+	}
+
 	// Upload a file (in XMPP case send the upload URL because XMPP has no native upload support).
+	var err error
 	if msg.Extra != nil {
 		for _, rmsg := range helper.HandleExtra(&msg, b.General) {
 			b.Log.Debugf("=> Sending attachement message %#v", rmsg)
-			if _, err := b.xc.Send(xmpp.Chat{
+			if b.GetString("WebhookURL") != "" {
-				Type:   "groupchat",
+				err = b.postSlackCompatibleWebhook(msg)
-				Remote: rmsg.Channel + "@" + b.GetString("Muc"),
+			} else {
-				Text:   rmsg.Username + rmsg.Text,
+				_, err = b.xc.Send(xmpp.Chat{
-			}); err != nil {
+					Type:   "groupchat",
+					Remote: rmsg.Channel + "@" + b.GetString("Muc"),
+					Text:   rmsg.Username + rmsg.Text,
+				})
+			}
+
+			if err != nil {
 				b.Log.WithError(err).Error("Unable to send message with share URL.")
 			}
 		}
@@ -86,13 +112,24 @@ func (b *Bxmpp) Send(msg config.Message) (string, error) {
 		}
 	}
 
+	if b.GetString("WebhookURL") != "" {
+		b.Log.Debugf("Sending message using Webhook")
+		err := b.postSlackCompatibleWebhook(msg)
+		if err != nil {
+			b.Log.Errorf("Failed to send message using webhook: %s", err)
+			return "", err
+		}
+
+		return "", nil
+	}
+
+	// Post normal message.
 	var msgReplaceID string
 	msgID := xid.New().String()
 	if msg.ID != "" {
 		msgID = msg.ID
 		msgReplaceID = msg.ID
 	}
-	// Post normal message.
 	b.Log.Debugf("=> Sending message %#v", msg)
 	if _, err := b.xc.Send(xmpp.Chat{
 		Type:      "groupchat",
@@ -106,6 +143,25 @@ func (b *Bxmpp) Send(msg config.Message) (string, error) {
 	return msgID, nil
 }
 
+func (b *Bxmpp) postSlackCompatibleWebhook(msg config.Message) error {
+	type XMPPWebhook struct {
+		Username string `json:"username"`
+		Text     string `json:"text"`
+	}
+	webhookBody, err := json.Marshal(XMPPWebhook{
+		Username: msg.Username,
+		Text:     msg.Text,
+	})
+	if err != nil {
+		b.Log.Errorf("Failed to marshal webhook: %s", err)
+		return err
+	}
+
+	resp, err := http.Post(b.GetString("WebhookURL")+"/"+msg.Channel, "application/json", bytes.NewReader(webhookBody))
+	resp.Body.Close()
+	return err
+}
+
 func (b *Bxmpp) createXMPP() error {
 	if !strings.Contains(b.GetString("Jid"), "@") {
 		return fmt.Errorf("the Jid %s doesn't contain an @", b.GetString("Jid"))
@@ -114,20 +170,22 @@ func (b *Bxmpp) createXMPP() error {
 		ServerName:         strings.Split(b.GetString("Jid"), "@")[1],
 		InsecureSkipVerify: b.GetBool("SkipTLSVerify"), // nolint: gosec
 	}
+
+	xmpp.DebugWriter = b.Log.Writer()
+
 	options := xmpp.Options{
 		Host:                         b.GetString("Server"),
 		User:                         b.GetString("Jid"),
 		Password:                     b.GetString("Password"),
 		NoTLS:                        true,
-		StartTLS:                     true,
+		StartTLS:                     !b.GetBool("NoTLS"),
 		TLSConfig:                    tc,
 		Debug:                        b.GetBool("debug"),
-		Logger:                       b.Log.Writer(),
 		Session:                      true,
 		Status:                       "",
 		StatusMessage:                "",
 		Resource:                     "",
-		InsecureAllowUnencryptedAuth: false,
+		InsecureAllowUnencryptedAuth: b.GetBool("NoTLS"),
 	}
 	var err error
 	b.xc, err = options.NewClient()
@@ -228,6 +286,16 @@ func (b *Bxmpp) handleXMPP() error {
 					event = config.EventTopicChange
 				}
 
+				available, sok := b.avatarAvailability[v.Remote]
+				avatar := ""
+				if !sok {
+					b.Log.Debugf("Requesting avatar data")
+					b.avatarAvailability[v.Remote] = false
+					b.xc.AvatarRequestData(v.Remote)
+				} else if available {
+					avatar = getAvatar(b.avatarMap, v.Remote, b.General)
+				}
+
 				msgID := v.ID
 				if v.ReplaceID != "" {
 					msgID = v.ReplaceID
@@ -237,6 +305,7 @@ func (b *Bxmpp) handleXMPP() error {
 					Text:     v.Text,
 					Channel:  b.parseChannel(v.Remote),
 					Account:  b.Account,
+					Avatar:   avatar,
 					UserID:   v.Remote,
 					ID:       msgID,
 					Event:    event,
@@ -253,6 +322,10 @@ func (b *Bxmpp) handleXMPP() error {
 				b.Log.Debugf("<= Message is %#v", rmsg)
 				b.Remote <- rmsg
 			}
+		case xmpp.AvatarData:
+			b.handleDownloadAvatar(v)
+			b.avatarAvailability[v.From] = true
+			b.Log.Debugf("Avatar for %s is now available", v.From)
 		case xmpp.Presence:
 			// Do nothing.
 		}
@@ -345,6 +418,11 @@ func (b *Bxmpp) skipMessage(message xmpp.Chat) bool {
 		return true
 	}
 
+	// Ignore messages posted by our webhook
+	if b.GetString("WebhookURL") != "" && strings.Contains(message.ID, "webhookbot") {
+		return true
+	}
+
 	// skip delayed messages
 	return !message.Stamp.IsZero() && time.Since(message.Stamp).Minutes() > 5
 }

bridge/zulip/zulip.go

@@ -135,19 +135,25 @@ func (b *Bzulip) handleQueue() error {
 			if m.SenderEmail == b.GetString("login") {
 				continue
 			}
+
+			avatarURL := m.AvatarURL
+			if !strings.HasPrefix(avatarURL, "http") {
+				avatarURL = b.GetString("server") + avatarURL
+			}
+
 			rmsg := config.Message{
 				Username: m.SenderFullName,
 				Text:     m.Content,
 				Channel:  b.getChannel(m.StreamID) + "/topic:" + m.Subject,
 				Account:  b.Account,
 				UserID:   strconv.Itoa(m.SenderID),
-				Avatar:   m.AvatarURL,
+				Avatar:   avatarURL,
 			}
 			b.Log.Debugf("<= Sending message from %s on %s to gateway", rmsg.Username, b.Account)
 			b.Log.Debugf("<= Message is %#v", rmsg)
 			b.Remote <- rmsg
-			b.q.LastEventID = m.ID
 		}
+
 		time.Sleep(time.Second * 3)
 	}
 }

changelog.md

@@ -1,3 +1,312 @@
+# v1.22.0
+
+Discord users using autowebhooks are encouraged to upgrade to this release.
+
+## New features
+
+- vk: new protocol added: Add vk support (#1245)
+- xmpp: Allow the XMPP bridge to use slack compatible webhooks (xmpp) (#1364)
+
+## Enhancements
+
+- telegram: Rename .oga audio files to .ogg (telegram) (#1349)
+- telegram: Add jpe as valid image filename extension (telegram) (#1360)
+- discord: Add an even more debug option (discord) (#1368)
+- general: Update vendor (#1384)
+
+## Bugfixes
+
+- discord: Pick up all the webhooks (discord) (#1383). Fixes #1353
+
+This release couldn't exist without the following contributors:
+@ivanik7, @Polynomdivision, @PeterDaveHello, @Humorhenker, @qaisjp
+
+# v1.21.0
+
+## Breaking Changes
+
+- discord: Remove WebhookURL support (discord) (#1323)
+
+`WebhookURL` global setting for discord is removed and will quit matterbridge.
+New `AutoWebhooks=true` setting, which will automatically use (and create, if they do not exist) webhooks inside specific channels. This only works if the bot has Manage Webhooks permission in bridged channels (global permission or as a channel permission override). Backwards compatibility with channel-specific webhooks. More info [here](https://github.com/42wim/matterbridge/blob/master/matterbridge.toml.sample#L862). 
+
+## New features
+
+- discord: Create webhooks automatically (#1323)
+- discord: Add threading support with token (discord) (#1342)
+- irc: Join on invite (irc). Fixes #1231 (#1306)
+- irc: Add support for stateless bridging via draft/relaymsg (irc) (#1339)
+- whatsapp: Add support for deleting messages (whatsapp) (#1316)
+- whatsapp: Handle video downloads (whatsapp) (#1316)
+- whatsapp: Handle audio downloads (whatsapp) (#1316)
+
+## Enhancements
+
+- general: Parse fencedcode in ParseMarkdown. Fixes #1127 (#1329)
+- discord: Refactor guild finding code (discord) (#1319)
+- discord: Add a prefix handler for unthreaded messages (discord) (#1346)
+- irc: Add support for irc to irc notice (irc). Fixes #754 (#1305)
+- irc: Make handlers run async (irc) (#1325)
+- matrix: Show mxids in case of clashing usernames (matrix) (#1309)
+- matrix: Implement ratelimiting (matrix). Fixes #1238 (#1326)
+- matrix: Mark messages as read (matrix). Fixes #1317 (#1328)
+- nctalk: Update go-nc-talk (nctalk) (#1333)
+- rocketchat: Update rocketchat vendor (#1327)
+- tengo: Add UserID to RemoteNickFormat and Tengo (#1308)
+- whatsapp: Retry until we have contacts (whatsapp). Fixes #1122 (#1304)
+- whatsapp: Refactor/cleanup code (whatsapp)
+- whatsapp: Refactor handleTextMessage (whatsapp)
+- whatsapp: Refactor image downloads (whatsapp)
+- whatsapp: Rename jfif to jpg (whatsapp). Fixes #1292
+
+## Bugfix
+
+- discord: Reject cross-channel message references (discord) (#1345)
+- mumble: Add nil checks to text message handling (mumble) (#1321)
+
+This release couldn't exist without the following contributors:
+@nightmared, @qaisjp, @jlu5, @wschwab, @gary-kim, @s3lph, @JeremyRand
+
+# v1.20.0
+
+## Breaking
+
+- matrix: Send the display name instead of the user name (matrix) (#1282)  
+  Matrix now sends the displayname if set instead of the username. If you want to keep the username, add  `UseUsername=true` to your matrix config. <https://github.com/42wim/matterbridge/wiki/Settings#useusername-1>
+- discord: Disable webhook editing (discord) (#1296)  
+  Because of issues with ratelimiting of webhook editing, this feature is now disabled. If you have multiple discord channels you bridge, you'll need to add a `webhookURL` to the `[gateway.inout.options]`. See <https://github.com/42wim/matterbridge/blob/master/matterbridge.toml.sample#L1864-L1870> for an example.
+
+## New features
+
+- general: Allow tengo to drop messages using msgDrop (#1272)
+- general: Update libraries (whatsapp,markdown,mattermost,ssh-chat)
+- irc: Add PingDelay option (irc) (#1269)
+- matrix: Allow message edits on matrix (#1286)
+- xmpp: add NoTLS option to allow plaintext XMPP connections (#1288)
+
+## Enhancements
+
+- discord: Edit messages via webhook (1287)
+- general: Add extra debug to log time spent sending a message per bridge (#1299)
+
+This release couldn't exist without the following contributors:
+@nightmared, @zhoreeq
+
+# v1.19.0
+
+## New features
+
+- mumble: new protocol added: Add Mumble support (#1245)
+- nctalk: Add support for downloading files (nctalk) (#1249)
+- nctalk: Append a suffix if user is a guest user (nctalk) (#1250)
+
+## Enhancements
+
+- irc: Add even more debug for irc (#1266)
+- matrix: Add username formatting for all events (matrix) (#1233)
+- matrix: Permit uploading files of other mimetypes (#1237)
+- whatsapp: Use vendored whatsapp version (#1258)
+- whatsapp: Add username for images from WhatsApp (#1232)
+
+This release couldn't exist without the following contributors:
+@Dellle, @42wim, @gary-kim, @s3lph, @BenWiederhake
+
+# v1.18.3
+
+## Enhancements
+
+- nctalk: Add TLSConfig to nctalk (#1195)
+- whatsapp: Handle broadcasts as groups in Whatsapp #1213
+- matrix: switch to upstream gomatrix #1219
+- api: support multiple websocket clients #1205
+
+## Bugfix
+
+- general: update vendor
+- zulip: Check location of avatarURL (zulip). Fixes #1214 (#1227)
+- nctalk: Fix issue with too many open files #1223
+- nctalk: Fix mentions #1222
+- nctalk: Fix message replays #1220
+
+This release couldn't exist without the following contributors:
+@gary-kim, @tilosp, @NikkyAI, @escoand, @42wim
+
+# v1.18.2
+
+## Bugfix
+
+- zulip: Fix error loop (zulip) (#1210)
+- whatsapp: Update whatsapp vendor and fix a panic (#1209)
+
+This release couldn't exist without the following contributors:
+@SuperSandro2000, @42wim
+
+# v1.18.1
+
+## New features
+
+- telegram: Support Telegram animated stickers (tgs) format (#1173). See https://github.com/42wim/matterbridge/wiki/Settings#mediaConverttgs for more info
+
+## Enhancements
+
+- matrix: Remove HTML formatting for push messages (#1188) (#1189)
+- mattermost: Use mattermost v5 module (#1192)
+
+## Bugfix
+
+- whatsapp: Handle panic in whatsapp. Fixes #1180 (#1184)
+- nctalk: Fix Nextcloud Talk connection failure (#1179)
+- matrix: Sleep when ratelimited on joins (matrix). Fixes #1201 (#1206)
+
+This release couldn't exist without the following contributors:
+@42wim, @BenWiederhake, @Dellle, @gary-kim
+
+# v1.18.0
+
+## New features
+
+- nctalk: new protocol added. Add Nextcloud Talk support #1167
+- general: Add an option to log into a file rather than stdout (#1168)
+- api: Add websocket to API (#970)
+
+## Enhancements
+
+- telegram: Fix MarkdownV2 support in Telegram (#1169)
+- whatsapp: Reload user information when a new contact is detected (whatsapp) (#1160)
+- api: Add sane RemoteNickFormat default for API (#1157)
+- irc: Skip gIRC built-in rate limiting (irc) (#1164)
+- irc: Only colour IRC nicks if there is one. (#1161)
+- docker: Combine runs to one layer (#1151)
+
+## Bugfix
+
+- general: Update dependencies for 1.18.0 release (#1175)
+
+Discord users are encouraged to upgrade, this release works with the move to the discord.com domain.
+
+This release couldn't exist without the following contributors:
+@42wim, @jlu5, @qaisjp, @TheHolyRoger, @SuperSandro2000, @gary-kim, @z3bra, @greenx, @haykam821, @nathanaelhoun
+
+# v1.17.5
+
+## Enhancements
+
+- irc: Add StripMarkdown option (irc). (#1145)
+- general: Increase debug logging with function,file and linenumber (#1147)
+- general: Update Dockerfile so inotify works (#1148)
+- matrix: Add an option to disable sending HTML to matrix. Fixes #1022 (#1135)
+- xmpp: Implement xep-0245 (xmpp). Closes #1137 (#1144)
+
+## Bugfix
+
+- discord: Fix #1120: replaceAction "_" crash (discord) (#1121)
+- discord: Fix #1049: missing space before embeds (discord) (#1124)
+- discord: Fix webhook EventUserAction messages being skipped (discord) (#1133)
+- matrix: Avoid creating invalid url when the user doesn't have an avatar (matrix) (#1130)
+- msteams: Ignore non-user messages (msteams). Fixes #1141 (#1149)
+- slack: Do not use webhooks when token is configured (slack) (fixes #1123) (#1134)
+- telegram: Fix forward from hidden users (telegram). Closes #1131 (#1143)
+- xmpp: Prevent re-requesting avatar data (xmpp) (#1117)
+
+This release couldn't exist without the following contributors:
+@qaisjp, @xnaas, @42wim, @Polynomdivision, @tfve
+
+# v1.17.4
+
+## Bugfix
+
+- general: Lowercase account names. Fixes #1108 (#1110)
+- msteams: Remove panics and retry polling on failure (msteams). Fixes #1104 (#1105
+- whatsapp: Update Rhymen/go-whatsapp. Fixes #1107 (#1109) (make whatsapp working again)
+- discord: Add an ID cache (discord). Fixes #1106 (#1111) (fix delete/edits with webhooks)
+
+# v1.17.3
+
+## Enhancements
+
+- xmpp: Implement User Avatar spoofing of XMPP users #1090
+- rocketchat: Relay Joins/Topic changes in RocketChat bridge (#1085)
+- irc: Add JoinDelay option (irc). Fixes #1084 (#1098)
+- slack: Clip too long messages on 3000 length (slack). Fixes #1081 (#1102)
+
+## Bugfix
+
+- general: Fix the behavior of ShowTopicChange and SyncTopic (#1086)
+- slack: Prevent image/message looping (slack). Fixes #1088 (#1096)
+- whatsapp: Ignore non-critical errors (whatsapp). Fixes #1094 (#1100)
+- irc: Add extra space before colon in attachments (irc). Fixes #1089 (#1101)
+
+This release couldn't exist without the following contributors:
+@42wim, @ldruschk, @qaisjp, @Polynomdivision
+
+# v1.17.2
+
+## Enhancements
+
+- slack: Update vendor slack-go/slack (#1068)
+- general: Update vendor d5/tengo (#1066)
+- general: Clarify terminology used in mapping group chat IDs to channels in config (#1079)
+
+## Bugfix
+
+- whatsapp: Update Rhymen/go-whatsapp vendor and whatsapp version (#1078). Fixes Media upload #1074
+- whatsapp: Reset start timestamp on reconnect (whatsapp). Fixes #1059 (#1064)
+
+This release couldn't exist without the following contributors:
+@42wim, @jheiselman
+
+# v1.17.1
+
+## Enhancements
+
+- docker: Remove build dependencies from final image (multistage build) #1057
+
+## Bugfix
+
+- general: Don't transmit typing events from ourselves #1056
+- general: Add support for build tags #1054
+- discord: Strip extra info from emotes (discord) #1052
+- msteams: fix macos build: Update vendor yaegashi/msgraph.go to v0.1.2 #1036
+- whatsapp: Update client version whatsapp. Fixes #1061 #1062
+
+This release couldn't exist without the following contributors:
+@awigen, @qaisjp, @42wim
+
+# v1.17.0
+
+## New features
+
+- msteams: new protocol added. Add initial Microsoft Teams support #967
+  See https://github.com/42wim/matterbridge/wiki/MS-Teams-setup for a complete walkthrough
+- discord: Add ability to procure avatars from the destination bridge #1000
+- matrix: Add support for avatars from matrix. #1007
+- general: support JSON and YAML config formats #1045
+
+## Enhancements
+
+- discord: Check only bridged channels for PermManageWebhooks #1001
+- irc: Be less lossy when throttling IRC messages #1004
+- keybase: updated library #1002, #1019
+- matrix: Rebase gomatrix vendor with upstream #1006
+- slack: Use upstream slack-go/slack again #1018
+- slack: Ignore ConnectingEvent #1041
+- slack: use blocks not attachments #1048
+- sshchat: Update vendor shazow/ssh-chat #1029
+- telegram: added markdownv2 mode for telegram #1037
+- whatsapp: Implement basic reconnect (whatsapp). Fixes #987 #1003
+
+## Bugfix
+
+- discord: Fix webhook permission checks sometimes failing #1043
+- discord: Fix #1027: warning when handling inbound webhooks #1044
+- discord: Fix duplicate separator on empty description/url (discord) #1035
+- matrix: Fix issue with underscores in links #999
+- slack: Fix #1039: messages sent to Slack being synced back #1046
+- telegram: Make avatars download work with mediaserverdownload (telegram). Fixes #920 
+
+This release couldn't exist without the following contributors:
+@qaisjp, @jakubgs, @burner1024, @notpushkin, @MartijnBraam, @42wim
+
 # v1.16.5
 
 - Fix version bump

ci/bintray.sh

@@ -1,30 +0,0 @@
-#!/usr/bin/env bash
-set -u -e -x -o pipefail
-
-go version | grep go1.13 || exit
-
-VERSION=$(git describe --tags)
-mkdir ci/binaries
-GOOS=windows GOARCH=amd64 go build -ldflags "-s -w -X main.githash=$(git log --pretty=format:'%h' -n 1)" -o ci/binaries/matterbridge-$VERSION-windows-amd64.exe
-GOOS=linux GOARCH=amd64 go build -ldflags "-s -w -X main.githash=$(git log --pretty=format:'%h' -n 1)" -o ci/binaries/matterbridge-$VERSION-linux-amd64
-GOOS=linux GOARCH=arm go build -ldflags "-s -w -X main.githash=$(git log --pretty=format:'%h' -n 1)" -o ci/binaries/matterbridge-$VERSION-linux-arm
-GOOS=darwin GOARCH=amd64 go build -ldflags "-s -w -X main.githash=$(git log --pretty=format:'%h' -n 1)" -o ci/binaries/matterbridge-$VERSION-darwin-amd64
-cd ci
-cat > deploy.json <<EOF
-{
-    "package": {
-        "name": "Matterbridge",
-        "repo": "nightly",
-        "subject": "42wim"
-    },
-    "version": {
-        "name": "$VERSION"
-    },
-    "files":
-        [
-        {"includePattern": "ci/binaries/(.*)", "uploadPattern":"\$1"}
-        ],
-    "publish": true
-}
-EOF
-

ci/lint.sh

@@ -1,17 +0,0 @@
-#!/usr/bin/env bash
-set -u -e -x -o pipefail
-
-if [[ -n "${GOLANGCI_VERSION-}" ]]; then
-  # Retrieve the golangci-lint linter binary.
-  curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | bash -s -- -b ${GOPATH}/bin ${GOLANGCI_VERSION}
-fi
-
-# Run the linter.
-golangci-lint run
-
-# if [[ "${GO111MODULE-off}" == "on" ]]; then
-#   # If Go modules are active then check that dependencies are correctly maintained.
-#   go mod tidy
-#   go mod vendor
-#   git diff --exit-code --quiet || (echo "Please run 'go mod tidy' to clean up the 'go.mod' and 'go.sum' files."; false)
-# fi

ci/test.sh

@@ -1,17 +0,0 @@
-#!/usr/bin/env bash
-set -u -e -x -o pipefail
-
-if [[ -n "${REPORT_COVERAGE+cover}" ]]; then
-  # Retrieve and prepare CodeClimate's test coverage reporter.
-  curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
-  chmod +x ./cc-test-reporter
-  ./cc-test-reporter before-build
-fi
-
-# Run all the tests with the race detector and generate coverage.
-go test -v -race -coverprofile c.out ./...
-
-if [[ -n "${REPORT_COVERAGE+cover}" && "${TRAVIS_SECURE_ENV_VARS}" == "true" ]]; then
-  # Upload test coverage to CodeClimate.
-  ./cc-test-reporter after-build
-fi

gateway/bridgemap/api.go

@@ -0,0 +1,11 @@
+// +build !noapi
+
+package bridgemap
+
+import (
+	"github.com/42wim/matterbridge/bridge/api"
+)
+
+func init() {
+	FullMap["api"] = api.New
+}

gateway/bridgemap/bdiscord.go

@@ -0,0 +1,12 @@
+// +build !nodiscord
+
+package bridgemap
+
+import (
+	bdiscord "github.com/42wim/matterbridge/bridge/discord"
+)
+
+func init() {
+	FullMap["discord"] = bdiscord.New
+	UserTypingSupport["discord"] = struct{}{}
+}

gateway/bridgemap/bgitter.go

@@ -0,0 +1,11 @@
+// +build !nogitter
+
+package bridgemap
+
+import (
+	bgitter "github.com/42wim/matterbridge/bridge/gitter"
+)
+
+func init() {
+	FullMap["gitter"] = bgitter.New
+}

gateway/bridgemap/birc.go

@@ -0,0 +1,11 @@
+// +build !noirc
+
+package bridgemap
+
+import (
+	birc "github.com/42wim/matterbridge/bridge/irc"
+)
+
+func init() {
+	FullMap["irc"] = birc.New
+}

gateway/bridgemap/bkeybase.go

@@ -0,0 +1,11 @@
+// +build !nokeybase
+
+package bridgemap
+
+import (
+	bkeybase "github.com/42wim/matterbridge/bridge/keybase"
+)
+
+func init() {
+	FullMap["keybase"] = bkeybase.New
+}

gateway/bridgemap/bmatrix.go

@@ -0,0 +1,11 @@
+// +build !nomatrix
+
+package bridgemap
+
+import (
+	bmatrix "github.com/42wim/matterbridge/bridge/matrix"
+)
+
+func init() {
+	FullMap["matrix"] = bmatrix.New
+}

gateway/bridgemap/bmattermost.go

@@ -0,0 +1,11 @@
+// +build !nomattermost
+
+package bridgemap
+
+import (
+	bmattermost "github.com/42wim/matterbridge/bridge/mattermost"
+)
+
+func init() {
+	FullMap["mattermost"] = bmattermost.New
+}

gateway/bridgemap/bmsteams.go

@@ -0,0 +1,11 @@
+// +build !nomsteams
+
+package bridgemap
+
+import (
+	bmsteams "github.com/42wim/matterbridge/bridge/msteams"
+)
+
+func init() {
+	FullMap["msteams"] = bmsteams.New
+}

gateway/bridgemap/bmumble.go

@@ -0,0 +1,11 @@
+// +build !nomumble
+
+package bridgemap
+
+import (
+	bmumble "github.com/42wim/matterbridge/bridge/mumble"
+)
+
+func init() {
+	FullMap["mumble"] = bmumble.New
+}

gateway/bridgemap/bnctalk.go

@@ -0,0 +1,11 @@
+// +build !nonctalk
+
+package bridgemap
+
+import (
+	btalk "github.com/42wim/matterbridge/bridge/nctalk"
+)
+
+func init() {
+	FullMap["nctalk"] = btalk.New
+}

gateway/bridgemap/bridgemap.go

@@ -2,45 +2,9 @@ package bridgemap
 
 import (
 	"github.com/42wim/matterbridge/bridge"
-	"github.com/42wim/matterbridge/bridge/api"
-	bdiscord "github.com/42wim/matterbridge/bridge/discord"
-	bgitter "github.com/42wim/matterbridge/bridge/gitter"
-	birc "github.com/42wim/matterbridge/bridge/irc"
-	bkeybase "github.com/42wim/matterbridge/bridge/keybase"
-	bmatrix "github.com/42wim/matterbridge/bridge/matrix"
-	bmattermost "github.com/42wim/matterbridge/bridge/mattermost"
-	brocketchat "github.com/42wim/matterbridge/bridge/rocketchat"
-	bslack "github.com/42wim/matterbridge/bridge/slack"
-	bsshchat "github.com/42wim/matterbridge/bridge/sshchat"
-	bsteam "github.com/42wim/matterbridge/bridge/steam"
-	btelegram "github.com/42wim/matterbridge/bridge/telegram"
-	bwhatsapp "github.com/42wim/matterbridge/bridge/whatsapp"
-	bxmpp "github.com/42wim/matterbridge/bridge/xmpp"
-	bzulip "github.com/42wim/matterbridge/bridge/zulip"
 )
 
 var (
-	FullMap = map[string]bridge.Factory{
+	FullMap           = map[string]bridge.Factory{}
-		"api":          api.New,
+	UserTypingSupport = map[string]struct{}{}
-		"discord":      bdiscord.New,
-		"gitter":       bgitter.New,
-		"irc":          birc.New,
-		"mattermost":   bmattermost.New,
-		"matrix":       bmatrix.New,
-		"rocketchat":   brocketchat.New,
-		"slack-legacy": bslack.NewLegacy,
-		"slack":        bslack.New,
-		"sshchat":      bsshchat.New,
-		"steam":        bsteam.New,
-		"telegram":     btelegram.New,
-		"whatsapp":     bwhatsapp.New,
-		"xmpp":         bxmpp.New,
-		"zulip":        bzulip.New,
-		"keybase":      bkeybase.New,
-	}
-
-	UserTypingSupport = map[string]struct{}{
-		"slack":   {},
-		"discord": {},
-	}
 )

gateway/bridgemap/brocketchat.go

@@ -0,0 +1,11 @@
+// +build !norocketchat
+
+package bridgemap
+
+import (
+	brocketchat "github.com/42wim/matterbridge/bridge/rocketchat"
+)
+
+func init() {
+	FullMap["rocketchat"] = brocketchat.New
+}

gateway/bridgemap/bslack.go

@@ -0,0 +1,13 @@
+// +build !noslack
+
+package bridgemap
+
+import (
+	bslack "github.com/42wim/matterbridge/bridge/slack"
+)
+
+func init() {
+	FullMap["slack-legacy"] = bslack.NewLegacy
+	FullMap["slack"] = bslack.New
+	UserTypingSupport["slack"] = struct{}{}
+}

gateway/bridgemap/bsshchat.go

@@ -0,0 +1,11 @@
+// +build !nosshchat
+
+package bridgemap
+
+import (
+	bsshchat "github.com/42wim/matterbridge/bridge/sshchat"
+)
+
+func init() {
+	FullMap["sshchat"] = bsshchat.New
+}

gateway/bridgemap/bsteam.go

@@ -0,0 +1,11 @@
+// +build !nosteam
+
+package bridgemap
+
+import (
+	bsteam "github.com/42wim/matterbridge/bridge/steam"
+)
+
+func init() {
+	FullMap["steam"] = bsteam.New
+}

gateway/bridgemap/btelegram.go

@@ -0,0 +1,11 @@
+// +build !notelegram
+
+package bridgemap
+
+import (
+	btelegram "github.com/42wim/matterbridge/bridge/telegram"
+)
+
+func init() {
+	FullMap["telegram"] = btelegram.New
+}

gateway/bridgemap/bvk.go

@@ -0,0 +1,11 @@
+// +build !novk
+
+package bridgemap
+
+import (
+	bvk "github.com/42wim/matterbridge/bridge/vk"
+)
+
+func init() {
+	FullMap["vk"] = bvk.New
+}

gateway/bridgemap/bwhatsapp.go

@@ -0,0 +1,11 @@
+// +build !nowhatsapp
+
+package bridgemap
+
+import (
+	bwhatsapp "github.com/42wim/matterbridge/bridge/whatsapp"
+)
+
+func init() {
+	FullMap["whatsapp"] = bwhatsapp.New
+}

gateway/bridgemap/bxmpp.go

@@ -0,0 +1,11 @@
+// +build !noxmpp
+
+package bridgemap
+
+import (
+	bxmpp "github.com/42wim/matterbridge/bridge/xmpp"
+)
+
+func init() {
+	FullMap["xmpp"] = bxmpp.New
+}

gateway/bridgemap/bzulip.go

@@ -0,0 +1,11 @@
+// +build !nozulip
+
+package bridgemap
+
+import (
+	bzulip "github.com/42wim/matterbridge/bridge/zulip"
+)
+
+func init() {
+	FullMap["zulip"] = bzulip.New
+}

gateway/gateway.go

@@ -1,6 +1,7 @@
 package gateway
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
 	"regexp"
@@ -108,7 +109,7 @@ func (gw *Gateway) AddBridge(cfg *config.Bridge) error {
 func (gw *Gateway) checkConfig(cfg *config.Bridge) {
 	match := false
 	for _, key := range gw.Router.Config.Viper().AllKeys() {
-		if strings.HasPrefix(key, cfg.Account) {
+		if strings.HasPrefix(key, strings.ToLower(cfg.Account)) {
 			match = true
 			break
 		}
@@ -306,8 +307,6 @@ func (gw *Gateway) ignoreMessage(msg *config.Message) bool {
 }
 
 func (gw *Gateway) modifyUsername(msg *config.Message, dest *bridge.Bridge) string {
-	br := gw.Bridges[msg.Account]
-	msg.Protocol = br.Protocol
 	if dest.GetBool("StripNick") {
 		re := regexp.MustCompile("[^a-zA-Z0-9]+")
 		msg.Username = re.ReplaceAllString(msg.Username, "")
@@ -315,6 +314,7 @@ func (gw *Gateway) modifyUsername(msg *config.Message, dest *bridge.Bridge) stri
 	nick := dest.GetString("RemoteNickFormat")
 
 	// loop to replace nicks
+	br := gw.Bridges[msg.Account]
 	for _, outer := range br.GetStringSlice2D("ReplaceNicks") {
 		search := outer[0]
 		replace := outer[1]
@@ -337,20 +337,21 @@ func (gw *Gateway) modifyUsername(msg *config.Message, dest *bridge.Bridge) stri
 			}
 			i++
 		}
-		nick = strings.Replace(nick, "{NOPINGNICK}", msg.Username[:i]+"​"+msg.Username[i:], -1)
+		nick = strings.ReplaceAll(nick, "{NOPINGNICK}", msg.Username[:i]+"\u200b"+msg.Username[i:])
 	}
 
-	nick = strings.Replace(nick, "{BRIDGE}", br.Name, -1)
+	nick = strings.ReplaceAll(nick, "{BRIDGE}", br.Name)
-	nick = strings.Replace(nick, "{PROTOCOL}", br.Protocol, -1)
+	nick = strings.ReplaceAll(nick, "{PROTOCOL}", br.Protocol)
-	nick = strings.Replace(nick, "{GATEWAY}", gw.Name, -1)
+	nick = strings.ReplaceAll(nick, "{GATEWAY}", gw.Name)
-	nick = strings.Replace(nick, "{LABEL}", br.GetString("Label"), -1)
+	nick = strings.ReplaceAll(nick, "{LABEL}", br.GetString("Label"))
-	nick = strings.Replace(nick, "{NICK}", msg.Username, -1)
+	nick = strings.ReplaceAll(nick, "{NICK}", msg.Username)
-	nick = strings.Replace(nick, "{CHANNEL}", msg.Channel, -1)
+	nick = strings.ReplaceAll(nick, "{USERID}", msg.UserID)
+	nick = strings.ReplaceAll(nick, "{CHANNEL}", msg.Channel)
 	tengoNick, err := gw.modifyUsernameTengo(msg, br)
 	if err != nil {
 		gw.logger.Errorf("modifyUsernameTengo error: %s", err)
 	}
-	nick = strings.Replace(nick, "{TENGO}", tengoNick, -1) //nolint:gocritic
+	nick = strings.ReplaceAll(nick, "{TENGO}", tengoNick)
 	return nick
 }
 
@@ -364,10 +365,23 @@ func (gw *Gateway) modifyAvatar(msg *config.Message, dest *bridge.Bridge) string
 }
 
 func (gw *Gateway) modifyMessage(msg *config.Message) {
-	if err := modifyMessageTengo(gw.BridgeValues().General.TengoModifyMessage, msg); err != nil {
+	if gw.BridgeValues().General.TengoModifyMessage != "" {
+		gw.logger.Warnf("General TengoModifyMessage=%s is deprecated and will be removed in v1.20.0, please move to Tengo InMessage=%s", gw.BridgeValues().General.TengoModifyMessage, gw.BridgeValues().General.TengoModifyMessage)
+	}
+
+	if err := modifyInMessageTengo(gw.BridgeValues().General.TengoModifyMessage, msg); err != nil {
 		gw.logger.Errorf("TengoModifyMessage failed: %s", err)
 	}
-	if err := modifyMessageTengo(gw.BridgeValues().Tengo.Message, msg); err != nil {
+
+	inMessage := gw.BridgeValues().Tengo.InMessage
+	if inMessage == "" {
+		inMessage = gw.BridgeValues().Tengo.Message
+		if inMessage != "" {
+			gw.logger.Warnf("Tengo Message=%s is deprecated and will be removed in v1.20.0, please move to Tengo InMessage=%s", inMessage, inMessage)
+		}
+	}
+
+	if err := modifyInMessageTengo(inMessage, msg); err != nil {
 		gw.logger.Errorf("Tengo.Message failed: %s", err)
 	}
 
@@ -417,9 +431,15 @@ func (gw *Gateway) SendMessage(
 		}
 	}
 
+	// Only send irc notices to irc
+	if msg.Event == config.EventNoticeIRC && dest.Protocol != "irc" {
+		return "", nil
+	}
+
 	// Too noisy to log like other events
+	debugSendMessage := ""
 	if msg.Event != config.EventUserTyping {
-		gw.logger.Debugf("=> Sending %#v from %s (%s) to %s (%s)", msg, msg.Account, rmsg.Channel, dest.Account, channel.Name)
+		debugSendMessage = fmt.Sprintf("=> Sending %#v from %s (%s) to %s (%s)", msg, msg.Account, rmsg.Channel, dest.Account, channel.Name)
 	}
 
 	msg.Channel = channel.Name
@@ -439,22 +459,34 @@ func (gw *Gateway) SendMessage(
 	}
 
 	// if the parentID is still empty and we have a parentID set in the original message
-	// this means that we didn't find it in the cache so set it "msg-parent-not-found"
+	// this means that we didn't find it in the cache so set it to a "msg-parent-not-found" constant
 	if msg.ParentID == "" && rmsg.ParentID != "" {
-		msg.ParentID = "msg-parent-not-found"
+		msg.ParentID = config.ParentIDNotFound
 	}
 
-	err := gw.modifySendMessageTengo(rmsg, &msg, dest)
+	drop, err := gw.modifyOutMessageTengo(rmsg, &msg, dest)
 	if err != nil {
 		gw.logger.Errorf("modifySendMessageTengo: %s", err)
 	}
 
+	if drop {
+		gw.logger.Debugf("=> Tengo dropping %#v from %s (%s) to %s (%s)", msg, msg.Account, rmsg.Channel, dest.Account, channel.Name)
+		return "", nil
+	}
+
+	if debugSendMessage != "" {
+		gw.logger.Debug(debugSendMessage)
+	}
 	// if we are using mattermost plugin account, send messages to MattermostPlugin channel
 	// that can be picked up by the mattermost matterbridge plugin
 	if dest.Account == "mattermost.plugin" {
 		gw.Router.MattermostPlugin <- msg
 	}
 
+	defer func(t time.Time) {
+		gw.logger.Debugf("=> Send from %s (%s) to %s (%s) took %s", msg.Account, rmsg.Channel, dest.Account, channel.Name, time.Since(t))
+	}(time.Now())
+
 	mID, err := dest.Send(msg)
 	if err != nil {
 		return mID, err
@@ -506,7 +538,7 @@ func getProtocol(msg *config.Message) string {
 	return p[0]
 }
 
-func modifyMessageTengo(filename string, msg *config.Message) error {
+func modifyInMessageTengo(filename string, msg *config.Message) error {
 	if filename == "" {
 		return nil
 	}
@@ -518,6 +550,7 @@ func modifyMessageTengo(filename string, msg *config.Message) error {
 	s.SetImports(stdlib.GetModuleMap(stdlib.AllModuleNames()...))
 	_ = s.Add("msgText", msg.Text)
 	_ = s.Add("msgUsername", msg.Username)
+	_ = s.Add("msgUserID", msg.UserID)
 	_ = s.Add("msgAccount", msg.Account)
 	_ = s.Add("msgChannel", msg.Channel)
 	c, err := s.Compile()
@@ -546,6 +579,7 @@ func (gw *Gateway) modifyUsernameTengo(msg *config.Message, br *bridge.Bridge) (
 	_ = s.Add("result", "")
 	_ = s.Add("msgText", msg.Text)
 	_ = s.Add("msgUsername", msg.Username)
+	_ = s.Add("msgUserID", msg.UserID)
 	_ = s.Add("nick", msg.Username)
 	_ = s.Add("msgAccount", msg.Account)
 	_ = s.Add("msgChannel", msg.Channel)
@@ -565,22 +599,28 @@ func (gw *Gateway) modifyUsernameTengo(msg *config.Message, br *bridge.Bridge) (
 	return c.Get("result").String(), nil
 }
 
-func (gw *Gateway) modifySendMessageTengo(origmsg *config.Message, msg *config.Message, br *bridge.Bridge) error {
+func (gw *Gateway) modifyOutMessageTengo(origmsg *config.Message, msg *config.Message, br *bridge.Bridge) (bool, error) {
 	filename := gw.BridgeValues().Tengo.OutMessage
-	var res []byte
+	var (
-	var err error
+		res  []byte
+		err  error
+		drop bool
+	)
+
 	if filename == "" {
 		res, err = internal.Asset("tengo/outmessage.tengo")
 		if err != nil {
-			return err
+			return drop, err
 		}
 	} else {
 		res, err = ioutil.ReadFile(filename)
 		if err != nil {
-			return err
+			return drop, err
 		}
 	}
+
 	s := tengo.NewScript(res)
+
 	s.SetImports(stdlib.GetModuleMap(stdlib.AllModuleNames()...))
 	_ = s.Add("inAccount", origmsg.Account)
 	_ = s.Add("inProtocol", origmsg.Protocol)
@@ -594,14 +634,20 @@ func (gw *Gateway) modifySendMessageTengo(origmsg *config.Message, msg *config.M
 	_ = s.Add("outEvent", msg.Event)
 	_ = s.Add("msgText", msg.Text)
 	_ = s.Add("msgUsername", msg.Username)
+	_ = s.Add("msgUserID", msg.UserID)
+	_ = s.Add("msgDrop", drop)
 	c, err := s.Compile()
 	if err != nil {
-		return err
+		return drop, err
 	}
+
 	if err := c.Run(); err != nil {
-		return err
+		return drop, err
 	}
+
+	drop = c.Get("msgDrop").Bool()
 	msg.Text = c.Get("msgText").String()
 	msg.Username = c.Get("msgUsername").String()
-	return nil
+
+	return drop, nil
 }

gateway/gateway_test.go

@@ -533,7 +533,7 @@ func (s *ignoreTestSuite) TestIgnoreNicks() {
 func BenchmarkTengo(b *testing.B) {
 	msg := &config.Message{Username: "user", Text: "blah testing", Account: "protocol.account", Channel: "mychannel"}
 	for n := 0; n < b.N; n++ {
-		err := modifyMessageTengo("bench.tengo", msg)
+		err := modifyInMessageTengo("bench.tengo", msg)
 		if err != nil {
 			return
 		}

gateway/handlers.go

@@ -169,7 +169,7 @@ func (gw *Gateway) ignoreEvent(event string, dest *bridge.Bridge) bool {
 	switch event {
 	case config.EventAvatarDownload:
 		// Avatar downloads are only relevant for telegram and mattermost for now
-		if dest.Protocol != "mattermost" && dest.Protocol != "telegram" {
+		if dest.Protocol != "mattermost" && dest.Protocol != "telegram" && dest.Protocol != "xmpp" {
 			return true
 		}
 	case config.EventJoinLeave:
@@ -179,7 +179,7 @@ func (gw *Gateway) ignoreEvent(event string, dest *bridge.Bridge) bool {
 		}
 	case config.EventTopicChange:
 		// only relay topic change when used in some way on other side
-		if dest.GetBool("ShowTopicChange") && dest.GetBool("SyncTopic") {
+		if !dest.GetBool("ShowTopicChange") && !dest.GetBool("SyncTopic") {
 			return true
 		}
 	}

gateway/router.go

@@ -132,6 +132,9 @@ func (r *Router) handleReceive() {
 		r.handleEventFailure(&msg)
 		r.handleEventRejoinChannels(&msg)
 
+		// Set message protocol based on the account it came from
+		msg.Protocol = r.getBridge(msg.Account).Protocol
+
 		filesHandled := false
 		for _, gw := range r.Gateways {
 			// record all the message ID's of the different bridges
@@ -157,7 +160,7 @@ func (r *Router) handleReceive() {
 				// For some bridges we always add/update the message ID.
 				// This is necessary as msgIDs will change if a bridge returns
 				// a different ID in response to edits.
-				if !exists || msg.Protocol == "discord" {
+				if !exists {
 					gw.Messages.Add(msg.Protocol+" "+msg.ID, msgIDs)
 				}
 			}

go.mod

@@ -3,60 +3,60 @@ module github.com/42wim/matterbridge
 require (
 	github.com/42wim/go-gitter v0.0.0-20170828205020-017310c2d557
 	github.com/Baozisoftware/qrcode-terminal-go v0.0.0-20170407111555-c0650d8dff0f
-	github.com/Jeffail/gabs v1.1.1 // indirect
+	github.com/Philipp15b/go-steam v1.0.1-0.20200727090957-6ae9b3c0a560
-	github.com/Philipp15b/go-steam v1.0.1-0.20190816133340-b04c5a83c1c0
+	github.com/Rhymen/go-whatsapp v0.1.2-0.20210126174449-3c094ebae0ce
-	github.com/Rhymen/go-whatsapp v0.1.0
+	github.com/RocketChat/Rocket.Chat.Go.SDK v0.0.0-20200922220614-e4a51dfb52e4 // indirect
-	github.com/bwmarrin/discordgo v0.20.2
+	github.com/SevereCloud/vksdk/v2 v2.9.0
-	github.com/d5/tengo/v2 v2.0.2
+	github.com/d5/tengo/v2 v2.6.2
-	github.com/dfordsoft/golib v0.0.0-20180902042739-76ee6ab99bec
+	github.com/davecgh/go-spew v1.1.1
-	github.com/fsnotify/fsnotify v1.4.7
+	github.com/fsnotify/fsnotify v1.4.9
-	github.com/go-telegram-bot-api/telegram-bot-api v4.6.5-0.20181225215658-ec221ba9ea45+incompatible
+	github.com/go-telegram-bot-api/telegram-bot-api v1.0.1-0.20200524105306-7434b0456e81
-	github.com/gomarkdown/markdown v0.0.0-20200127000047-1813ea067497
+	github.com/gomarkdown/markdown v0.0.0-20201113031856-722100d81a8e
-	github.com/google/gops v0.3.6
+	github.com/google/gops v0.3.14
-	github.com/gopackage/ddp v0.0.0-20170117053602-652027933df4 // indirect
+	github.com/gorilla/schema v1.2.0
-	github.com/gorilla/schema v1.1.0
+	github.com/gorilla/websocket v1.4.2
-	github.com/gorilla/websocket v1.4.1
+	github.com/hashicorp/golang-lru v0.5.4
-	github.com/hashicorp/golang-lru v0.5.3
-	github.com/hpcloud/tail v1.0.0 // indirect
 	github.com/jpillora/backoff v1.0.0
-	github.com/keybase/go-keybase-chat-bot v0.0.0-20190816161829-561f10822eb2
+	github.com/keybase/go-keybase-chat-bot v0.0.0-20200505163032-5cacf52379da
-	github.com/labstack/echo/v4 v4.1.13
+	github.com/labstack/echo/v4 v4.1.17
 	github.com/lrstanley/girc v0.0.0-20190801035559-4fc93959e1a7
-	github.com/matterbridge/Rocket.Chat.Go.SDK v0.0.0-20190210153444-cc9d05784d5d
+	github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd
+	github.com/matterbridge/Rocket.Chat.Go.SDK v0.0.0-20201206215757-c1d86d75b9f8
+	github.com/matterbridge/discordgo v0.21.2-0.20210201201054-fb39a175b4f7
 	github.com/matterbridge/emoji v2.1.1-0.20191117213217-af507f6b02db+incompatible
-	github.com/matterbridge/go-xmpp v0.0.0-20180529212104-cd19799fba91
+	github.com/matterbridge/go-xmpp v0.0.0-20200418225040-c8a3a57b4050
-	github.com/matterbridge/gomatrix v0.0.0-20191026211822-6fc7accd00ca
+	github.com/matterbridge/gozulipbot v0.0.0-20200820220548-be5824faa913
-	github.com/matterbridge/gozulipbot v0.0.0-20190212232658-7aa251978a18
+	github.com/matterbridge/logrus-prefixed-formatter v0.5.3-0.20200523233437-d971309a77ba
-	github.com/matterbridge/logrus-prefixed-formatter v0.0.0-20180806162718-01618749af61
+	github.com/mattermost/mattermost-server/v5 v5.30.1
-	github.com/mattermost/mattermost-server v5.5.0+incompatible
+	github.com/mattn/godown v0.0.1
 	github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect
+	github.com/missdeer/golib v1.0.4
+	github.com/mitchellh/mapstructure v1.3.3 // indirect
 	github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474 // indirect
 	github.com/mrexodia/wray v0.0.0-20160318003008-78a2c1f284ff // indirect
 	github.com/nelsonken/gomf v0.0.0-20180504123937-a9dd2f9deae9
-	github.com/nicksnyder/go-i18n v1.4.0 // indirect
-	github.com/nlopes/slack v0.6.0
-	github.com/onsi/ginkgo v1.6.0 // indirect
-	github.com/onsi/gomega v1.4.1 // indirect
 	github.com/paulrosania/go-charset v0.0.0-20190326053356-55c9d7a5834c
-	github.com/pborman/uuid v0.0.0-20160216163710-c55201b03606 // indirect
 	github.com/rs/xid v1.2.1
-	github.com/russross/blackfriday v1.5.2
+	github.com/russross/blackfriday v1.6.0
 	github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca
-	github.com/shazow/ssh-chat v1.8.2
+	github.com/shazow/ssh-chat v1.10.1
-	github.com/sirupsen/logrus v1.4.2
+	github.com/sirupsen/logrus v1.7.0
-	github.com/spf13/viper v1.6.1
+	github.com/slack-go/slack v0.8.0
-	github.com/stretchr/testify v1.4.0
+	github.com/spf13/afero v1.3.4 // indirect
-	github.com/technoweenie/multipartstreamer v1.0.1 // indirect
+	github.com/spf13/cast v1.3.1 // indirect
+	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/spf13/viper v1.7.1
+	github.com/stretchr/testify v1.7.0
+	github.com/vincent-petithory/dataurl v0.0.0-20191104211930-d1553a71de50
+	github.com/writeas/go-strip-markdown v2.0.1+incompatible
 	github.com/x-cray/logrus-prefixed-formatter v0.5.2 // indirect
-	github.com/zfjagann/golang-ring v0.0.0-20190106091943-a88bb6aef447
+	github.com/yaegashi/msgraph.go v0.1.4
-	golang.org/x/image v0.0.0-20191214001246-9130b4cfad52
+	github.com/zfjagann/golang-ring v0.0.0-20190304061218-d34796e0a6c2
-	gopkg.in/fsnotify.v1 v1.4.7 // indirect
+	golang.org/x/image v0.0.0-20201208152932-35266b937fa6
-	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
+	golang.org/x/oauth2 v0.0.0-20210201163806-010130855d6c
-	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
+	gomod.garykim.dev/nc-talk v0.1.7
+	gopkg.in/olahol/melody.v1 v1.0.0-20170518105555-d52139073376
+	layeh.com/gumble v0.0.0-20200818122324-146f9205029b
 )
 
-replace github.com/nlopes/slack v0.6.0 => github.com/matterbridge/slack v0.1.1-0.20191208194820-95190f11bfb6
+go 1.15
-
-replace github.com/bwmarrin/discordgo v0.20.2 => github.com/matterbridge/discordgo v0.18.1-0.20200109173909-ed873362fa43
-
-go 1.13

matterbridge.go

@@ -4,6 +4,7 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"runtime"
 	"strings"
 
 	"github.com/42wim/matterbridge/bridge/config"
@@ -15,7 +16,7 @@ import (
 )
 
 var (
-	version = "1.16.5"
+	version = "1.22.0"
 	githash string
 
 	flagConfig  = flag.String("conf", "matterbridge.toml", "config file")
@@ -50,6 +51,15 @@ func main() {
 	cfg := config.NewConfig(rootLogger, *flagConfig)
 	cfg.BridgeValues().General.Debug = *flagDebug
 
+	// if logging to a file, ensure it is closed when the program terminates
+	// nolint:errcheck
+	defer func() {
+		if f, ok := rootLogger.Out.(*os.File); ok {
+			f.Sync()
+			f.Close()
+		}
+	}()
+
 	r, err := gateway.NewRouter(rootLogger, cfg, bridgemap.FullMap)
 	if err != nil {
 		logger.Fatalf("Starting gateway failed: %s", err)
@@ -67,17 +77,31 @@ func setupLogger() *logrus.Logger {
 		Formatter: &prefixed.TextFormatter{
 			PrefixPadding: 13,
 			DisableColors: true,
-			FullTimestamp: true,
 		},
 		Level: logrus.InfoLevel,
 	}
 	if *flagDebug || os.Getenv("DEBUG") == "1" {
+		logger.SetReportCaller(true)
 		logger.Formatter = &prefixed.TextFormatter{
-			PrefixPadding:   13,
+			PrefixPadding: 13,
-			DisableColors:   true,
+			DisableColors: true,
-			FullTimestamp:   false,
+			FullTimestamp: false,
-			ForceFormatting: true,
+
+			CallerFormatter: func(function, file string) string {
+				return fmt.Sprintf(" [%s:%s]", function, file)
+			},
+			CallerPrettyfier: func(f *runtime.Frame) (string, string) {
+				sp := strings.SplitAfter(f.File, "/matterbridge/")
+				filename := f.File
+				if len(sp) > 1 {
+					filename = sp[1]
+				}
+				s := strings.Split(f.Function, ".")
+				funcName := s[len(s)-1]
+				return funcName, fmt.Sprintf("%s:%d", filename, f.Line)
+			},
 		}
+
 		logger.Level = logrus.DebugLevel
 		logger.WithFields(logrus.Fields{"prefix": "main"}).Info("Enabling debug logging.")
 	}

matterclient/channels.go

@@ -4,7 +4,7 @@ import (
 	"errors"
 	"strings"
 
-	"github.com/mattermost/mattermost-server/model"
+	"github.com/mattermost/mattermost-server/v5/model"
 )
 
 // GetChannels returns all channels we're members off
@@ -167,7 +167,7 @@ func (m *MMClient) JoinChannel(channelId string) error { //nolint:golint
 }
 
 func (m *MMClient) UpdateChannelsTeam(teamID string) error {
-	mmchannels, resp := m.Client.GetChannelsForTeamForUser(teamID, m.User.Id, "")
+	mmchannels, resp := m.Client.GetChannelsForTeamForUser(teamID, m.User.Id, false, "")
 	if resp.Error != nil {
 		return errors.New(resp.Error.DetailedError)
 	}

matterclient/helpers.go

@@ -13,7 +13,7 @@ import (
 
 	"github.com/gorilla/websocket"
 	"github.com/jpillora/backoff"
-	"github.com/mattermost/mattermost-server/model"
+	"github.com/mattermost/mattermost-server/v5/model"
 )
 
 func (m *MMClient) doLogin(firstConnection bool, b *backoff.Backoff) error {
@@ -154,7 +154,7 @@ func (m *MMClient) initUser() error {
 
 		t := &Team{Team: team, Users: usermap, Id: team.Id}
 
-		mmchannels, resp := m.Client.GetChannelsForTeamForUser(team.Id, m.User.Id, "")
+		mmchannels, resp := m.Client.GetChannelsForTeamForUser(team.Id, m.User.Id, false, "")
 		if resp.Error != nil {
 			return resp.Error
 		}

matterclient/matterclient.go

@@ -11,7 +11,7 @@ import (
 	lru "github.com/hashicorp/golang-lru"
 	"github.com/jpillora/backoff"
 	prefixed "github.com/matterbridge/logrus-prefixed-formatter"
-	"github.com/mattermost/mattermost-server/model"
+	"github.com/mattermost/mattermost-server/v5/model"
 	"github.com/sirupsen/logrus"
 )
 
@@ -69,6 +69,7 @@ type MMClient struct {
 	logger     *logrus.Entry
 	rootLogger *logrus.Logger
 	lruCache   *lru.Cache
+	allevents  bool
 }
 
 // New will instantiate a new Matterclient with the specified login details without connecting.
@@ -119,6 +120,10 @@ func (m *MMClient) SetLogLevel(level string) {
 	}
 }
 
+func (m *MMClient) EnableAllEvents() {
+	m.allevents = true
+}
+
 // Login tries to connect the client with the loging details with which it was initialized.
 func (m *MMClient) Login() error {
 	// check if this is a first connect or a reconnection
@@ -220,6 +225,10 @@ func (m *MMClient) WsReceiver() {
 					continue
 				}
 			}
+			if m.allevents {
+				m.MessageChan <- msg
+				continue
+			}
 			switch msg.Raw.Event {
 			case model.WEBSOCKET_EVENT_USER_ADDED,
 				model.WEBSOCKET_EVENT_USER_REMOVED,

matterclient/messages.go

@@ -3,7 +3,7 @@ package matterclient
 import (
 	"strings"
 
-	"github.com/mattermost/mattermost-server/model"
+	"github.com/mattermost/mattermost-server/v5/model"
 )
 
 func (m *MMClient) parseActionPost(rmsg *Message) {

matterclient/users.go

@@ -4,7 +4,7 @@ import (
 	"errors"
 	"time"
 
-	"github.com/mattermost/mattermost-server/model"
+	"github.com/mattermost/mattermost-server/v5/model"
 )
 
 func (m *MMClient) GetNickName(userId string) string { //nolint:golint

matterhook/matterhook.go

@@ -14,7 +14,7 @@ import (
 	"time"
 
 	"github.com/gorilla/schema"
-	"github.com/nlopes/slack"
+	"github.com/slack-go/slack"
 )
 
 // OMessage for mattermost incoming webhook. (send to mattermost)

tgs.Dockerfile

@@ -0,0 +1,38 @@
+FROM alpine AS builder
+
+COPY . /go/src/github.com/42wim/matterbridge
+RUN apk add \
+    go \
+    git \
+    gcc \
+    musl-dev \
+  && cd /go/src/github.com/42wim/matterbridge \
+  && export GOPATH=/go \
+  && go get \
+  && go build -x -ldflags "-X main.githash=$(git log --pretty=format:'%h' -n 1)" -o /bin/matterbridge
+
+FROM alpine
+RUN apk --no-cache add \
+    ca-certificates \
+    cairo \
+    libjpeg-turbo \
+    mailcap \
+    py3-webencodings \
+    python3 \
+  && apk --no-cache add --virtual .compile \
+    gcc \
+    libffi-dev \
+    libjpeg-turbo-dev \
+    musl-dev \
+    py3-pip \
+    py3-wheel \
+    python3-dev \
+    zlib-dev \
+  && pip3 install --no-cache-dir lottie[PNG] \
+  && apk --no-cache del .compile
+
+COPY --from=builder /bin/matterbridge /bin/matterbridge
+RUN mkdir /etc/matterbridge \
+  && touch /etc/matterbridge/matterbridge.toml \
+  && ln -sf /matterbridge.toml /etc/matterbridge/matterbridge.toml
+ENTRYPOINT ["/bin/matterbridge", "-conf", "/etc/matterbridge/matterbridge.toml"]

vendor/github.com/Jeffail/gabs/README.md

@@ -7,21 +7,17 @@ It does nothing spectacular except for being fabulous.
 
 https://godoc.org/github.com/Jeffail/gabs
 
-## How to install:
+## Install
 
 ``` bash
 go get github.com/Jeffail/gabs
 ```
 
-## How to use
+## Use
 
 ### Parsing and searching JSON
 
 ``` go
-...
-
-import "github.com/Jeffail/gabs"
-
 jsonParsed, err := gabs.ParseJSON([]byte(`{
 	"outter":{
 		"inner":{
@@ -29,7 +25,10 @@ jsonParsed, err := gabs.ParseJSON([]byte(`{
 			"value2":22
 		},
 		"alsoInner":{
-			"value1":20
+			"value1":20,
+			"array1":[
+				30, 40
+			]
 		}
 	}
 }`))
@@ -43,26 +42,26 @@ value, ok = jsonParsed.Path("outter.inner.value1").Data().(float64)
 value, ok = jsonParsed.Search("outter", "inner", "value1").Data().(float64)
 // value == 10.0, ok == true
 
+gObj, err := jsonParsed.JSONPointer("/outter/alsoInner/array1/1")
+if err != nil {
+	panic(err)
+}
+value, ok = gObj.Data().(float64)
+// value == 40.0, ok == true
+
 value, ok = jsonParsed.Path("does.not.exist").Data().(float64)
 // value == 0.0, ok == false
 
 exists := jsonParsed.Exists("outter", "inner", "value1")
 // exists == true
 
-exists := jsonParsed.Exists("does", "not", "exist")
-// exists == false
-
 exists := jsonParsed.ExistsP("does.not.exist")
 // exists == false
-
-...
 ```
 
 ### Iterating objects
 
 ``` go
-...
-
 jsonParsed, _ := gabs.ParseJSON([]byte(`{"object":{ "first": 1, "second": 2, "third": 3 }}`))
 
 // S is shorthand for Search
@@ -70,24 +69,25 @@ children, _ := jsonParsed.S("object").ChildrenMap()
 for key, child := range children {
 	fmt.Printf("key: %v, value: %v\n", key, child.Data().(string))
 }
-
-...
 ```
 
 ### Iterating arrays
 
 ``` go
-...
+jsonParsed, err := gabs.ParseJSON([]byte(`{"array":[ "first", "second", "third" ]}`))
-
+if err != nil {
-jsonParsed, _ := gabs.ParseJSON([]byte(`{"array":[ "first", "second", "third" ]}`))
+	panic(err)
+}
 
 // S is shorthand for Search
-children, _ := jsonParsed.S("array").Children()
+children, err := jsonParsed.S("array").Children()
+if err != nil {
+	panic(err)
+}
+
 for _, child := range children {
 	fmt.Println(child.Data().(string))
 }
-
-...
 ```
 
 Will print:
@@ -108,12 +108,11 @@ objects within the array, this returns a JSON array containing the results for
 each element.
 
 ``` go
-...
+jsonParsed, err := gabs.ParseJSON([]byte(`{"array":[ {"value":1}, {"value":2}, {"value":3} ]}`))
-
+if err != nil {
-jsonParsed, _ := gabs.ParseJSON([]byte(`{"array":[ {"value":1}, {"value":2}, {"value":3} ]}`))
+	panic(err)
+}
 fmt.Println(jsonParsed.Path("array.value").String())
-
-...
 ```
 
 Will print:
@@ -125,8 +124,6 @@ Will print:
 ### Generating JSON
 
 ``` go
-...
-
 jsonObj := gabs.New()
 // or gabs.Consume(jsonObject) to work on an existing map[string]interface{}
 
@@ -135,8 +132,6 @@ jsonObj.SetP(20, "outter.inner.value2")
 jsonObj.Set(30, "outter", "inner2", "value3")
 
 fmt.Println(jsonObj.String())
-
-...
 ```
 
 Will print:
@@ -148,11 +143,7 @@ Will print:
 To pretty-print:
 
 ``` go
-...
-
 fmt.Println(jsonObj.StringIndent("", "  "))
-
-...
 ```
 
 Will print:
@@ -174,8 +165,6 @@ Will print:
 ### Generating Arrays
 
 ``` go
-...
-
 jsonObj := gabs.New()
 
 jsonObj.Array("foo", "array")
@@ -186,8 +175,6 @@ jsonObj.ArrayAppend(20, "foo", "array")
 jsonObj.ArrayAppend(30, "foo", "array")
 
 fmt.Println(jsonObj.String())
-
-...
 ```
 
 Will print:
@@ -199,8 +186,6 @@ Will print:
 Working with arrays by index:
 
 ``` go
-...
-
 jsonObj := gabs.New()
 
 // Create an array with the length of 3
@@ -217,8 +202,6 @@ jsonObj.S("foo").Index(2).SetIndex(2, 1)
 jsonObj.S("foo").Index(2).SetIndex(3, 2)
 
 fmt.Println(jsonObj.String())
-
-...
 ```
 
 Will print:
@@ -232,8 +215,6 @@ Will print:
 This is the easiest part:
 
 ``` go
-...
-
 jsonParsedObj, _ := gabs.ParseJSON([]byte(`{
 	"outter":{
 		"values":{
@@ -246,15 +227,11 @@ jsonParsedObj, _ := gabs.ParseJSON([]byte(`{
 
 jsonOutput := jsonParsedObj.String()
 // Becomes `{"outter":{"values":{"first":10,"second":11}},"outter2":"hello world"}`
-
-...
 ```
 
 And to serialize a specific segment is as simple as:
 
 ``` go
-...
-
 jsonParsedObj := gabs.ParseJSON([]byte(`{
 	"outter":{
 		"values":{
@@ -267,8 +244,6 @@ jsonParsedObj := gabs.ParseJSON([]byte(`{
 
 jsonOutput := jsonParsedObj.Search("outter").String()
 // Becomes `{"values":{"first":10,"second":11}}`
-
-...
 ```
 
 ### Merge two containers

vendor/github.com/Jeffail/gabs/gabs.go

@@ -20,58 +20,85 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 */
 
-// Package gabs implements a simplified wrapper around creating and parsing JSON.
+// Package gabs implements a simplified wrapper around creating and parsing
+// unknown or dynamic JSON.
 package gabs
 
 import (
 	"bytes"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"io"
 	"io/ioutil"
+	"strconv"
 	"strings"
 )
 
-//--------------------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
 
 var (
-	// ErrOutOfBounds - Index out of bounds.
+	// ErrOutOfBounds indicates an index was out of bounds.
 	ErrOutOfBounds = errors.New("out of bounds")
 
-	// ErrNotObjOrArray - The target is not an object or array type.
+	// ErrNotObjOrArray is returned when a target is not an object or array type
+	// but needs to be for the intended operation.
 	ErrNotObjOrArray = errors.New("not an object or array")
 
-	// ErrNotObj - The target is not an object type.
+	// ErrNotObj is returned when a target is not an object but needs to be for
+	// the intended operation.
 	ErrNotObj = errors.New("not an object")
 
-	// ErrNotArray - The target is not an array type.
+	// ErrNotArray is returned when a target is not an array but needs to be for
+	// the intended operation.
 	ErrNotArray = errors.New("not an array")
 
-	// ErrPathCollision - Creating a path failed because an element collided with an existing value.
+	// ErrPathCollision is returned when creating a path failed because an
+	// element collided with an existing value.
 	ErrPathCollision = errors.New("encountered value collision whilst building path")
 
-	// ErrInvalidInputObj - The input value was not a map[string]interface{}.
+	// ErrInvalidInputObj is returned when the input value was not a
+	// map[string]interface{}.
 	ErrInvalidInputObj = errors.New("invalid input object")
 
-	// ErrInvalidInputText - The input data could not be parsed.
+	// ErrInvalidInputText is returned when the input data could not be parsed.
 	ErrInvalidInputText = errors.New("input text could not be parsed")
 
-	// ErrInvalidPath - The filepath was not valid.
+	// ErrInvalidPath is returned when the filepath was not valid.
 	ErrInvalidPath = errors.New("invalid file path")
 
-	// ErrInvalidBuffer - The input buffer contained an invalid JSON string
+	// ErrInvalidBuffer is returned when the input buffer contained an invalid
+	// JSON string.
 	ErrInvalidBuffer = errors.New("input buffer contained invalid JSON")
 )
 
-//--------------------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
 
-// Container - an internal structure that holds a reference to the core interface map of the parsed
+func resolveJSONPointerHierarchy(path string) ([]string, error) {
-// json. Use this container to move context.
+	if len(path) < 1 {
+		return nil, errors.New("failed to resolve JSON pointer: path must not be empty")
+	}
+	if path[0] != '/' {
+		return nil, errors.New("failed to resolve JSON pointer: path must begin with '/'")
+	}
+	hierarchy := strings.Split(path, "/")[1:]
+	for i, v := range hierarchy {
+		v = strings.Replace(v, "~1", "/", -1)
+		v = strings.Replace(v, "~0", "~", -1)
+		hierarchy[i] = v
+	}
+	return hierarchy, nil
+}
+
+//------------------------------------------------------------------------------
+
+// Container references a specific element within a JSON structure.
 type Container struct {
 	object interface{}
 }
 
-// Data - Return the contained data as an interface{}.
+// Data returns the underlying interface{} of the target element in the JSON
+// structure.
 func (g *Container) Data() interface{} {
 	if g == nil {
 		return nil
@@ -79,17 +106,18 @@ func (g *Container) Data() interface{} {
 	return g.object
 }
 
-//--------------------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
 
-// Path - Search for a value using dot notation.
+// Path searches the JSON structure following a path in dot notation.
 func (g *Container) Path(path string) *Container {
 	return g.Search(strings.Split(path, ".")...)
 }
 
-// Search - Attempt to find and return an object within the JSON structure by specifying the
+// Search attempts to find and return an object within the JSON structure by
-// hierarchy of field names to locate the target. If the search encounters an array and has not
+// following a provided hierarchy of field names to locate the target. If the
-// reached the end target then it will iterate each object of the array for the target and return
+// search encounters an array and has not reached the end target then it will
-// all of the results in a JSON array.
+// iterate each object of the array for the target and return all of the results
+// in a JSON array.
 func (g *Container) Search(hierarchy ...string) *Container {
 	var object interface{}
 
@@ -120,22 +148,55 @@ func (g *Container) Search(hierarchy ...string) *Container {
 	return &Container{object}
 }
 
-// S - Shorthand method, does the same thing as Search.
+// JSONPointer parses a JSON pointer path (https://tools.ietf.org/html/rfc6901)
+// and either returns a *gabs.Container containing the result or an error if the
+// referenced item could not be found.
+func (g *Container) JSONPointer(path string) (*Container, error) {
+	hierarchy, err := resolveJSONPointerHierarchy(path)
+	if err != nil {
+		return nil, err
+	}
+
+	object := g.Data()
+	for target := 0; target < len(hierarchy); target++ {
+		pathSeg := hierarchy[target]
+		if mmap, ok := object.(map[string]interface{}); ok {
+			object, ok = mmap[pathSeg]
+			if !ok {
+				return nil, fmt.Errorf("failed to resolve JSON pointer: index '%v' value '%v' was not found", target, pathSeg)
+			}
+		} else if marray, ok := object.([]interface{}); ok {
+			index, err := strconv.Atoi(pathSeg)
+			if err != nil {
+				return nil, fmt.Errorf("failed to resolve JSON pointer: could not parse index '%v' value '%v' into array index: %v", target, pathSeg, err)
+			}
+			if len(marray) <= index {
+				return nil, fmt.Errorf("failed to resolve JSON pointer: index '%v' value '%v' exceeded target array size of '%v'", target, pathSeg, len(marray))
+			}
+			object = marray[index]
+		} else {
+			return &Container{nil}, fmt.Errorf("failed to resolve JSON pointer: index '%v' field '%v' was not found", target, pathSeg)
+		}
+	}
+	return &Container{object}, nil
+}
+
+// S is a shorthand alias for Search.
 func (g *Container) S(hierarchy ...string) *Container {
 	return g.Search(hierarchy...)
 }
 
-// Exists - Checks whether a path exists.
+// Exists checks whether a path exists.
 func (g *Container) Exists(hierarchy ...string) bool {
 	return g.Search(hierarchy...) != nil
 }
 
-// ExistsP - Checks whether a dot notation path exists.
+// ExistsP checks whether a dot notation path exists.
 func (g *Container) ExistsP(path string) bool {
 	return g.Exists(strings.Split(path, ".")...)
 }
 
-// Index - Attempt to find and return an object within a JSON array by index.
+// Index attempts to find and return an element within a JSON array by an index.
 func (g *Container) Index(index int) *Container {
 	if array, ok := g.Data().([]interface{}); ok {
 		if index >= len(array) {
@@ -146,9 +207,9 @@ func (g *Container) Index(index int) *Container {
 	return &Container{nil}
 }
 
-// Children - Return a slice of all the children of the array. This also works for objects, however,
+// Children returns a slice of all children of an array element. This also works
-// the children returned for an object will NOT be in order and you lose the names of the returned
+// for objects, however, the children returned for an object will be in a random
-// objects this way.
+// order and you lose the names of the returned objects this way.
 func (g *Container) Children() ([]*Container, error) {
 	if array, ok := g.Data().([]interface{}); ok {
 		children := make([]*Container, len(array))
@@ -167,7 +228,7 @@ func (g *Container) Children() ([]*Container, error) {
 	return nil, ErrNotObjOrArray
 }
 
-// ChildrenMap - Return a map of all the children of an object.
+// ChildrenMap returns a map of all the children of an object element.
 func (g *Container) ChildrenMap() (map[string]*Container, error) {
 	if mmap, ok := g.Data().(map[string]interface{}); ok {
 		children := map[string]*Container{}
@@ -179,11 +240,11 @@ func (g *Container) ChildrenMap() (map[string]*Container, error) {
 	return nil, ErrNotObj
 }
 
-//--------------------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
 
-// Set - Set the value of a field at a JSON path, any parts of the path that do not exist will be
+// Set the value of a field at a JSON path, any parts of the path that do not
-// constructed, and if a collision occurs with a non object type whilst iterating the path an error
+// exist will be constructed, and if a collision occurs with a non object type
-// is returned.
+// whilst iterating the path an error is returned.
 func (g *Container) Set(value interface{}, path ...string) (*Container, error) {
 	if len(path) == 0 {
 		g.object = value
@@ -209,12 +270,14 @@ func (g *Container) Set(value interface{}, path ...string) (*Container, error) {
 	return &Container{object}, nil
 }
 
-// SetP - Does the same as Set, but using a dot notation JSON path.
+// SetP sets the value of a field at a JSON path using dot notation, any parts
+// of the path that do not exist will be constructed, and if a collision occurs
+// with a non object type whilst iterating the path an error is returned.
 func (g *Container) SetP(value interface{}, path string) (*Container, error) {
 	return g.Set(value, strings.Split(path, ".")...)
 }
 
-// SetIndex - Set a value of an array element based on the index.
+// SetIndex attempts to set a value of an array element based on an index.
 func (g *Container) SetIndex(value interface{}, index int) (*Container, error) {
 	if array, ok := g.Data().([]interface{}); ok {
 		if index >= len(array) {
@@ -226,60 +289,112 @@ func (g *Container) SetIndex(value interface{}, index int) (*Container, error) {
 	return &Container{nil}, ErrNotArray
 }
 
-// Object - Create a new JSON object at a path. Returns an error if the path contains a collision
+// SetJSONPointer parses a JSON pointer path
-// with a non object type.
+// (https://tools.ietf.org/html/rfc6901) and sets the leaf to a value. Returns
+// an error if the pointer could not be resolved due to missing fields.
+func (g *Container) SetJSONPointer(value interface{}, path string) error {
+	hierarchy, err := resolveJSONPointerHierarchy(path)
+	if err != nil {
+		return err
+	}
+
+	if len(hierarchy) == 0 {
+		g.object = value
+		return nil
+	}
+
+	object := g.object
+
+	for target := 0; target < len(hierarchy); target++ {
+		pathSeg := hierarchy[target]
+		if mmap, ok := object.(map[string]interface{}); ok {
+			if target == len(hierarchy)-1 {
+				object = value
+				mmap[pathSeg] = object
+			} else if object = mmap[pathSeg]; object == nil {
+				return fmt.Errorf("failed to resolve JSON pointer: index '%v' value '%v' was not found", target, pathSeg)
+			}
+		} else if marray, ok := object.([]interface{}); ok {
+			index, err := strconv.Atoi(pathSeg)
+			if err != nil {
+				return fmt.Errorf("failed to resolve JSON pointer: could not parse index '%v' value '%v' into array index: %v", target, pathSeg, err)
+			}
+			if len(marray) <= index {
+				return fmt.Errorf("failed to resolve JSON pointer: index '%v' value '%v' exceeded target array size of '%v'", target, pathSeg, len(marray))
+			}
+			if target == len(hierarchy)-1 {
+				object = value
+				marray[index] = object
+			} else if object = marray[index]; object == nil {
+				return fmt.Errorf("failed to resolve JSON pointer: index '%v' value '%v' was not found", target, pathSeg)
+			}
+		} else {
+			return fmt.Errorf("failed to resolve JSON pointer: index '%v' value '%v' was not found", target, pathSeg)
+		}
+	}
+	return nil
+}
+
+// Object creates a new JSON object at a target path. Returns an error if the
+// path contains a collision with a non object type.
 func (g *Container) Object(path ...string) (*Container, error) {
 	return g.Set(map[string]interface{}{}, path...)
 }
 
-// ObjectP - Does the same as Object, but using a dot notation JSON path.
+// ObjectP creates a new JSON object at a target path using dot notation.
+// Returns an error if the path contains a collision with a non object type.
 func (g *Container) ObjectP(path string) (*Container, error) {
 	return g.Object(strings.Split(path, ".")...)
 }
 
-// ObjectI - Create a new JSON object at an array index. Returns an error if the object is not an
+// ObjectI creates a new JSON object at an array index. Returns an error if the
-// array or the index is out of bounds.
+// object is not an array or the index is out of bounds.
 func (g *Container) ObjectI(index int) (*Container, error) {
 	return g.SetIndex(map[string]interface{}{}, index)
 }
 
-// Array - Create a new JSON array at a path. Returns an error if the path contains a collision with
+// Array creates a new JSON array at a path. Returns an error if the path
-// a non object type.
+// contains a collision with a non object type.
 func (g *Container) Array(path ...string) (*Container, error) {
 	return g.Set([]interface{}{}, path...)
 }
 
-// ArrayP - Does the same as Array, but using a dot notation JSON path.
+// ArrayP creates a new JSON array at a path using dot notation. Returns an
+// error if the path contains a collision with a non object type.
 func (g *Container) ArrayP(path string) (*Container, error) {
 	return g.Array(strings.Split(path, ".")...)
 }
 
-// ArrayI - Create a new JSON array at an array index. Returns an error if the object is not an
+// ArrayI creates a new JSON array within an array at an index. Returns an error
-// array or the index is out of bounds.
+// if the element is not an array or the index is out of bounds.
 func (g *Container) ArrayI(index int) (*Container, error) {
 	return g.SetIndex([]interface{}{}, index)
 }
 
-// ArrayOfSize - Create a new JSON array of a particular size at a path. Returns an error if the
+// ArrayOfSize creates a new JSON array of a particular size at a path. Returns
-// path contains a collision with a non object type.
+// an error if the path contains a collision with a non object type.
 func (g *Container) ArrayOfSize(size int, path ...string) (*Container, error) {
 	a := make([]interface{}, size)
 	return g.Set(a, path...)
 }
 
-// ArrayOfSizeP - Does the same as ArrayOfSize, but using a dot notation JSON path.
+// ArrayOfSizeP creates a new JSON array of a particular size at a path using
+// dot notation. Returns an error if the path contains a collision with a non
+// object type.
 func (g *Container) ArrayOfSizeP(size int, path string) (*Container, error) {
 	return g.ArrayOfSize(size, strings.Split(path, ".")...)
 }
 
-// ArrayOfSizeI - Create a new JSON array of a particular size at an array index. Returns an error
+// ArrayOfSizeI create a new JSON array of a particular size within an array at
-// if the object is not an array or the index is out of bounds.
+// an index. Returns an error if the element is not an array or the index is out
+// of bounds.
 func (g *Container) ArrayOfSizeI(size, index int) (*Container, error) {
 	a := make([]interface{}, size)
 	return g.SetIndex(a, index)
 }
 
-// Delete - Delete an element at a JSON path, an error is returned if the element does not exist.
+// Delete an element at a path, an error is returned if the element does not
+// exist.
 func (g *Container) Delete(path ...string) error {
 	var object interface{}
 
@@ -304,45 +419,40 @@ func (g *Container) Delete(path ...string) error {
 	return nil
 }
 
-// DeleteP - Does the same as Delete, but using a dot notation JSON path.
+// DeleteP deletes an element at a path using dot notation, an error is returned
+// if the element does not exist.
 func (g *Container) DeleteP(path string) error {
 	return g.Delete(strings.Split(path, ".")...)
 }
 
-// Merge - Merges two gabs-containers
+// MergeFn merges two objects using a provided function to resolve collisions.
-func (g *Container) Merge(toMerge *Container) error {
+//
+// The collision function receives two interface{} arguments, destination (the
+// original object) and source (the object being merged into the destination).
+// Which ever value is returned becomes the new value in the destination object
+// at the location of the collision.
+func (g *Container) MergeFn(source *Container, collisionFn func(destination, source interface{}) interface{}) error {
 	var recursiveFnc func(map[string]interface{}, []string) error
 	recursiveFnc = func(mmap map[string]interface{}, path []string) error {
 		for key, value := range mmap {
 			newPath := append(path, key)
 			if g.Exists(newPath...) {
-				target := g.Search(newPath...)
+				existingData := g.Search(newPath...).Data()
 				switch t := value.(type) {
 				case map[string]interface{}:
-					switch targetV := target.Data().(type) {
+					switch existingVal := existingData.(type) {
 					case map[string]interface{}:
 						if err := recursiveFnc(t, newPath); err != nil {
 							return err
 						}
-					case []interface{}:
-						g.Set(append(targetV, t), newPath...)
 					default:
-						newSlice := append([]interface{}{}, targetV)
+						if _, err := g.Set(collisionFn(existingVal, t), newPath...); err != nil {
-						g.Set(append(newSlice, t), newPath...)
-					}
-				case []interface{}:
-					for _, valueOfSlice := range t {
-						if err := g.ArrayAppend(valueOfSlice, newPath...); err != nil {
 							return err
 						}
 					}
 				default:
-					switch targetV := target.Data().(type) {
+					if _, err := g.Set(collisionFn(existingData, t), newPath...); err != nil {
-					case []interface{}:
+						return err
-						g.Set(append(targetV, t), newPath...)
-					default:
-						newSlice := append([]interface{}{}, targetV)
-						g.Set(append(newSlice, t), newPath...)
 					}
 				}
 			} else {
@@ -354,21 +464,48 @@ func (g *Container) Merge(toMerge *Container) error {
 		}
 		return nil
 	}
-	if mmap, ok := toMerge.Data().(map[string]interface{}); ok {
+	if mmap, ok := source.Data().(map[string]interface{}); ok {
 		return recursiveFnc(mmap, []string{})
 	}
 	return nil
 }
 
-//--------------------------------------------------------------------------------------------------
+// Merge a source object into an existing destination object. When a collision
+// is found within the merged structures (both a source and destination object
+// contain the same non-object keys) the result will be an array containing both
+// values, where values that are already arrays will be expanded into the
+// resulting array.
+//
+// It is possible to merge structures will different collision behaviours with
+// MergeFn.
+func (g *Container) Merge(source *Container) error {
+	return g.MergeFn(source, func(dest, source interface{}) interface{} {
+		destArr, destIsArray := dest.([]interface{})
+		sourceArr, sourceIsArray := source.([]interface{})
+		if destIsArray {
+			if sourceIsArray {
+				return append(destArr, sourceArr...)
+			}
+			return append(destArr, source)
+		}
+		if sourceIsArray {
+			return append(append([]interface{}{}, dest), sourceArr...)
+		}
+		return []interface{}{dest, source}
+	})
+}
+
+//------------------------------------------------------------------------------
 
 /*
-Array modification/search - Keeping these options simple right now, no need for anything more
+Array modification/search - Keeping these options simple right now, no need for
-complicated since you can just cast to []interface{}, modify and then reassign with Set.
+anything more complicated since you can just cast to []interface{}, modify and
+then reassign with Set.
 */
 
-// ArrayAppend - Append a value onto a JSON array. If the target is not a JSON array then it will be
+// ArrayAppend attempts to append a value onto a JSON array at a path. If the
-// converted into one, with its contents as the first element of the array.
+// target is not a JSON array then it will be converted into one, with its
+// original contents set to the first element of the array.
 func (g *Container) ArrayAppend(value interface{}, path ...string) error {
 	if array, ok := g.Search(path...).Data().([]interface{}); ok {
 		array = append(array, value)
@@ -386,12 +523,15 @@ func (g *Container) ArrayAppend(value interface{}, path ...string) error {
 	return err
 }
 
-// ArrayAppendP - Append a value onto a JSON array using a dot notation JSON path.
+// ArrayAppendP attempts to append a value onto a JSON array at a path using dot
+// notation. If the target is not a JSON array then it will be converted into
+// one, with its original contents set to the first element of the array.
 func (g *Container) ArrayAppendP(value interface{}, path string) error {
 	return g.ArrayAppend(value, strings.Split(path, ".")...)
 }
 
-// ArrayRemove - Remove an element from a JSON array.
+// ArrayRemove attempts to remove an element identified by an index from a JSON
+// array at a path.
 func (g *Container) ArrayRemove(index int, path ...string) error {
 	if index < 0 {
 		return ErrOutOfBounds
@@ -409,12 +549,14 @@ func (g *Container) ArrayRemove(index int, path ...string) error {
 	return err
 }
 
-// ArrayRemoveP - Remove an element from a JSON array using a dot notation JSON path.
+// ArrayRemoveP attempts to remove an element identified by an index from a JSON
+// array at a path using dot notation.
 func (g *Container) ArrayRemoveP(index int, path string) error {
 	return g.ArrayRemove(index, strings.Split(path, ".")...)
 }
 
-// ArrayElement - Access an element from a JSON array.
+// ArrayElement attempts to access an element by an index from a JSON array at a
+// path.
 func (g *Container) ArrayElement(index int, path ...string) (*Container, error) {
 	if index < 0 {
 		return &Container{nil}, ErrOutOfBounds
@@ -429,12 +571,13 @@ func (g *Container) ArrayElement(index int, path ...string) (*Container, error)
 	return &Container{nil}, ErrOutOfBounds
 }
 
-// ArrayElementP - Access an element from a JSON array using a dot notation JSON path.
+// ArrayElementP attempts to access an element by an index from a JSON array at
+// a path using dot notation.
 func (g *Container) ArrayElementP(index int, path string) (*Container, error) {
 	return g.ArrayElement(index, strings.Split(path, ".")...)
 }
 
-// ArrayCount - Count the number of elements in a JSON array.
+// ArrayCount counts the number of elements in a JSON array at a path.
 func (g *Container) ArrayCount(path ...string) (int, error) {
 	if array, ok := g.Search(path...).Data().([]interface{}); ok {
 		return len(array), nil
@@ -442,14 +585,15 @@ func (g *Container) ArrayCount(path ...string) (int, error) {
 	return 0, ErrNotArray
 }
 
-// ArrayCountP - Count the number of elements in a JSON array using a dot notation JSON path.
+// ArrayCountP counts the number of elements in a JSON array at a path using dot
+// notation.
 func (g *Container) ArrayCountP(path string) (int, error) {
 	return g.ArrayCount(strings.Split(path, ".")...)
 }
 
-//--------------------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
 
-// Bytes - Converts the contained object back to a JSON []byte blob.
+// Bytes marshals an element to a JSON []byte blob.
 func (g *Container) Bytes() []byte {
 	if g.Data() != nil {
 		if bytes, err := json.Marshal(g.object); err == nil {
@@ -459,7 +603,8 @@ func (g *Container) Bytes() []byte {
 	return []byte("{}")
 }
 
-// BytesIndent - Converts the contained object to a JSON []byte blob formatted with prefix, indent.
+// BytesIndent marshals an element to a JSON []byte blob formatted with a prefix
+// and indent string.
 func (g *Container) BytesIndent(prefix string, indent string) []byte {
 	if g.object != nil {
 		if bytes, err := json.MarshalIndent(g.object, prefix, indent); err == nil {
@@ -469,12 +614,13 @@ func (g *Container) BytesIndent(prefix string, indent string) []byte {
 	return []byte("{}")
 }
 
-// String - Converts the contained object to a JSON formatted string.
+// String marshals an element to a JSON formatted string.
 func (g *Container) String() string {
 	return string(g.Bytes())
 }
 
-// StringIndent - Converts the contained object back to a JSON formatted string with prefix, indent.
+// StringIndent marshals an element to a JSON string formatted with a prefix and
+// indent string.
 func (g *Container) StringIndent(prefix string, indent string) string {
 	return string(g.BytesIndent(prefix, indent))
 }
@@ -496,10 +642,9 @@ func EncodeOptIndent(prefix string, indent string) EncodeOpt {
 	}
 }
 
-// EncodeJSON - Encodes the contained object back to a JSON formatted []byte
+// EncodeJSON marshals an element to a JSON formatted []byte using a variant
-// using a variant list of modifier functions for the encoder being used.
+// list of modifier functions for the encoder being used. Functions for
-// Functions for modifying the output are prefixed with EncodeOpt, e.g.
+// modifying the output are prefixed with EncodeOpt, e.g. EncodeOptHTMLEscape.
-// EncodeOptHTMLEscape.
 func (g *Container) EncodeJSON(encodeOpts ...EncodeOpt) []byte {
 	var b bytes.Buffer
 	encoder := json.NewEncoder(&b)
@@ -517,17 +662,18 @@ func (g *Container) EncodeJSON(encodeOpts ...EncodeOpt) []byte {
 	return result
 }
 
-// New - Create a new gabs JSON object.
+// New creates a new gabs JSON object.
 func New() *Container {
 	return &Container{map[string]interface{}{}}
 }
 
-// Consume - Gobble up an already converted JSON object, or a fresh map[string]interface{} object.
+// Consume an already unmarshalled JSON object (or a new map[string]interface{})
+// into a *Container.
 func Consume(root interface{}) (*Container, error) {
 	return &Container{root}, nil
 }
 
-// ParseJSON - Convert a string into a representation of the parsed JSON.
+// ParseJSON unmarshals a JSON byte slice into a *Container.
 func ParseJSON(sample []byte) (*Container, error) {
 	var gabs Container
 
@@ -538,7 +684,7 @@ func ParseJSON(sample []byte) (*Container, error) {
 	return &gabs, nil
 }
 
-// ParseJSONDecoder - Convert a json.Decoder into a representation of the parsed JSON.
+// ParseJSONDecoder applies a json.Decoder to a *Container.
 func ParseJSONDecoder(decoder *json.Decoder) (*Container, error) {
 	var gabs Container
 
@@ -549,7 +695,7 @@ func ParseJSONDecoder(decoder *json.Decoder) (*Container, error) {
 	return &gabs, nil
 }
 
-// ParseJSONFile - Read a file and convert into a representation of the parsed JSON.
+// ParseJSONFile reads a file and unmarshals the contents into a *Container.
 func ParseJSONFile(path string) (*Container, error) {
 	if len(path) > 0 {
 		cBytes, err := ioutil.ReadFile(path)
@@ -567,7 +713,7 @@ func ParseJSONFile(path string) (*Container, error) {
 	return nil, ErrInvalidPath
 }
 
-// ParseJSONBuffer - Read the contents of a buffer into a representation of the parsed JSON.
+// ParseJSONBuffer reads a buffer and unmarshals the contents into a *Container.
 func ParseJSONBuffer(buffer io.Reader) (*Container, error) {
 	var gabs Container
 	jsonDecoder := json.NewDecoder(buffer)
@@ -578,4 +724,4 @@ func ParseJSONBuffer(buffer io.Reader) (*Container, error) {
 	return &gabs, nil
 }
 
-//--------------------------------------------------------------------------------------------------
+//------------------------------------------------------------------------------

vendor/github.com/Jeffail/gabs/go.mod

@@ -0,0 +1 @@
+module github.com/Jeffail/gabs

vendor/github.com/Philipp15b/go-steam/.gitmodules

@@ -1,3 +1,6 @@
 [submodule "generator/SteamKit"]
 	path = generator/SteamKit
-	url = https://github.com/Philipp15b/SteamKit.git
+	url = https://github.com/SteamRE/SteamKit.git
+[submodule "generator/Protobufs"]
+	path = generator/Protobufs
+	url = https://github.com/SteamDatabase/Protobufs.git

vendor/github.com/Philipp15b/go-steam/auth.go

@@ -119,7 +119,7 @@ func (a *Auth) handleLogOnResponse(packet *Packet) {
 			ExtendedResult:            EResult(body.GetEresultExtended()),
 			OutOfGameSecsPerHeartbeat: body.GetOutOfGameHeartbeatSeconds(),
 			InGameSecsPerHeartbeat:    body.GetInGameHeartbeatSeconds(),
-			PublicIp:                  body.GetPublicIp(),
+			PublicIp:                  body.GetDeprecatedPublicIp(),
 			ServerTime:                body.GetRtime32ServerTime(),
 			AccountFlags:              EAccountFlags(body.GetAccountFlags()),
 			ClientSteamId:             SteamId(body.GetClientSuppliedSteamid()),
@@ -127,7 +127,7 @@ func (a *Auth) handleLogOnResponse(packet *Packet) {
 			CellId:                    body.GetCellId(),
 			CellIdPingThreshold:       body.GetCellIdPingThreshold(),
 			Steam2Ticket:              body.GetSteam2Ticket(),
-			UsePics:                   body.GetUsePics(),
+			UsePics:                   body.GetDeprecatedUsePics(),
 			WebApiUserNonce:           body.GetWebapiAuthenticateUserNonce(),
 			IpCountryCode:             body.GetIpCountryCode(),
 			VanityUrl:                 body.GetVanityUrl(),

vendor/github.com/Philipp15b/go-steam/go.mod

@@ -0,0 +1,8 @@
+module github.com/Philipp15b/go-steam
+
+go 1.14
+
+require (
+	github.com/davecgh/go-spew v1.1.1
+	github.com/golang/protobuf v1.4.2
+)

vendor/github.com/Philipp15b/go-steam/go.sum

@@ -0,0 +1,22 @@
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=

vendor/github.com/Philipp15b/go-steam/protocol/protobuf/app_ticket.pb.go

@@ -21,11 +21,11 @@ var _ = math.Inf
 const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
 
 type EncryptedAppTicket struct {
-	TicketVersionNo               *uint32  `protobuf:"varint,1,opt,name=ticket_version_no" json:"ticket_version_no,omitempty"`
+	TicketVersionNo               *uint32  `protobuf:"varint,1,opt,name=ticket_version_no,json=ticketVersionNo" json:"ticket_version_no,omitempty"`
-	CrcEncryptedticket            *uint32  `protobuf:"varint,2,opt,name=crc_encryptedticket" json:"crc_encryptedticket,omitempty"`
+	CrcEncryptedticket            *uint32  `protobuf:"varint,2,opt,name=crc_encryptedticket,json=crcEncryptedticket" json:"crc_encryptedticket,omitempty"`
-	CbEncrypteduserdata           *uint32  `protobuf:"varint,3,opt,name=cb_encrypteduserdata" json:"cb_encrypteduserdata,omitempty"`
+	CbEncrypteduserdata           *uint32  `protobuf:"varint,3,opt,name=cb_encrypteduserdata,json=cbEncrypteduserdata" json:"cb_encrypteduserdata,omitempty"`
-	CbEncryptedAppownershipticket *uint32  `protobuf:"varint,4,opt,name=cb_encrypted_appownershipticket" json:"cb_encrypted_appownershipticket,omitempty"`
+	CbEncryptedAppownershipticket *uint32  `protobuf:"varint,4,opt,name=cb_encrypted_appownershipticket,json=cbEncryptedAppownershipticket" json:"cb_encrypted_appownershipticket,omitempty"`
-	EncryptedTicket               []byte   `protobuf:"bytes,5,opt,name=encrypted_ticket" json:"encrypted_ticket,omitempty"`
+	EncryptedTicket               []byte   `protobuf:"bytes,5,opt,name=encrypted_ticket,json=encryptedTicket" json:"encrypted_ticket,omitempty"`
 	XXX_NoUnkeyedLiteral          struct{} `json:"-"`
 	XXX_unrecognized              []byte   `json:"-"`
 	XXX_sizecache                 int32    `json:"-"`
@@ -98,16 +98,18 @@ func init() {
 func init() { proto.RegisterFile("encrypted_app_ticket.proto", fileDescriptor_c6d69fd1cac4e8d5) }
 
 var fileDescriptor_c6d69fd1cac4e8d5 = []byte{
-	// 164 bytes of a gzipped FileDescriptorProto
+	// 202 bytes of a gzipped FileDescriptorProto
 	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x4a, 0xcd, 0x4b, 0x2e,
 	0xaa, 0x2c, 0x28, 0x49, 0x4d, 0x89, 0x4f, 0x2c, 0x28, 0x88, 0x2f, 0xc9, 0x4c, 0xce, 0x4e, 0x2d,
-	0xd1, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x57, 0x5a, 0xcb, 0xc8, 0x25, 0xe4, 0x0a, 0x93, 0x76, 0x2c,
+	0xd1, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x57, 0x9a, 0xce, 0xc4, 0x25, 0xe4, 0x0a, 0x93, 0x76, 0x2c,
-	0x28, 0x08, 0x01, 0x4b, 0x0a, 0x49, 0x72, 0x09, 0x42, 0x94, 0xc5, 0x97, 0xa5, 0x16, 0x15, 0x67,
+	0x28, 0x08, 0x01, 0x4b, 0x0a, 0x69, 0x71, 0x09, 0x42, 0x94, 0xc5, 0x97, 0xa5, 0x16, 0x15, 0x67,
-	0xe6, 0xe7, 0xc5, 0xe7, 0xe5, 0x4b, 0x30, 0x2a, 0x30, 0x6a, 0xf0, 0x0a, 0x49, 0x73, 0x09, 0x27,
+	0xe6, 0xe7, 0xc5, 0xe7, 0xe5, 0x4b, 0x30, 0x2a, 0x30, 0x6a, 0xf0, 0x06, 0xf1, 0x43, 0x24, 0xc2,
-	0x17, 0x25, 0xc7, 0xc3, 0xcd, 0x84, 0xa8, 0x93, 0x60, 0x02, 0x4b, 0xca, 0x70, 0x89, 0x24, 0x27,
+	0x20, 0xe2, 0x7e, 0xf9, 0x42, 0xfa, 0x5c, 0xc2, 0xc9, 0x45, 0xc9, 0xf1, 0x70, 0x4b, 0x20, 0xf2,
-	0x21, 0xe4, 0x4a, 0x8b, 0x53, 0x8b, 0x52, 0x12, 0x4b, 0x12, 0x25, 0x98, 0xc1, 0xb2, 0xea, 0x5c,
+	0x12, 0x4c, 0x60, 0xd5, 0x42, 0xc9, 0x45, 0xc9, 0xae, 0xa8, 0x32, 0x42, 0x86, 0x5c, 0x22, 0xc9,
-	0xf2, 0xc8, 0xb2, 0x20, 0xd7, 0xe4, 0x97, 0xe7, 0xa5, 0x16, 0x15, 0x67, 0x64, 0x16, 0x40, 0x8d,
+	0x49, 0x08, 0xf5, 0xa5, 0xc5, 0xa9, 0x45, 0x29, 0x89, 0x25, 0x89, 0x12, 0xcc, 0x60, 0x1d, 0xc2,
-	0x61, 0x01, 0x2b, 0x94, 0xe0, 0x12, 0x40, 0xa8, 0x82, 0xca, 0xb0, 0x2a, 0x30, 0x6a, 0xf0, 0x38,
+	0xc9, 0x49, 0xae, 0xe8, 0x52, 0x42, 0x6e, 0x5c, 0xf2, 0xc8, 0x5a, 0x40, 0xfe, 0xc8, 0x2f, 0xcf,
-	0xb1, 0x7a, 0x30, 0x36, 0x30, 0x32, 0x00, 0x02, 0x00, 0x00, 0xff, 0xff, 0x03, 0x8c, 0xdb, 0x92,
+	0x4b, 0x2d, 0x2a, 0xce, 0xc8, 0x2c, 0x80, 0xda, 0xc7, 0x02, 0xd6, 0x2d, 0x8b, 0xa4, 0xdb, 0x11,
-	0xd3, 0x00, 0x00, 0x00,
+	0x43, 0x91, 0x90, 0x26, 0x97, 0x00, 0xc2, 0x10, 0xa8, 0x46, 0x56, 0x05, 0x46, 0x0d, 0x9e, 0x20,
+	0x7e, 0xb8, 0x38, 0x24, 0x08, 0x9c, 0x58, 0x3d, 0x18, 0x1b, 0x18, 0x19, 0x00, 0x01, 0x00, 0x00,
+	0xff, 0xff, 0xe8, 0x03, 0x98, 0x21, 0x3d, 0x01, 0x00, 0x00,
 }