Release v1.23.2 (#1631)

* Detect errors when working with AvatarData

* Remove not neccessary line

Co-authored-by: Wim <wim@42.be>

.dockerignore

@@ -0,0 +1,2 @@
+Dockerfile
+tgs.Dockerfile

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,71 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+name: "CodeQL"
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [master]
+  schedule:
+    - cron: '0 16 * * 1'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        # Override automatic language detection by changing the below list
+        # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python']
+        language: ['go']
+        # Learn more...
+        # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+      with:
+        # We must fetch at least the immediate parents so that if this is
+        # a pull request then we can checkout the head.
+        fetch-depth: 2
+
+    # If this run was triggered by a pull request event, then checkout
+    # the head of the pull request instead of the merge commit.
+    - run: git checkout HEAD^2
+      if: ${{ github.event_name == 'pull_request' }}
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file. 
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/development.yml

@@ -9,14 +9,14 @@ jobs:
         with:
           fetch-depth: 20
       - name: Run golangci-lint
-        uses: golangci/golangci-lint-action@v1
+        uses: golangci/golangci-lint-action@v2
         with:
-          version: v1.27
+          version: latest
-          args: "-v --new-from-rev HEAD~1"
+          args: "-v --new-from-rev HEAD~5"
   test-build-upload:
     strategy:
       matrix:
-        go-version: [1.13.x, 1.14.x]
+        go-version: [1.17.x]
         platform: [ubuntu-latest]
     runs-on: ${{ matrix.platform }}
     steps:
@@ -24,6 +24,7 @@ jobs:
       uses: actions/setup-go@v2
       with:
         go-version: ${{ matrix.go-version }}
+        stable: false
     - name: Checkout code
       uses: actions/checkout@v2
       with:
@@ -34,23 +35,23 @@ jobs:
       run: |
         mkdir -p output/{win,lin,arm,mac}
         VERSION=$(git describe --tags)
-        GOOS=linux GOARCH=amd64 go build -mod=vendor -ldflags "-s -X main.githash=$(git log --pretty=format:'%h' -n 1)" -o output/lin/matterbridge-$VERSION-linux-amd64
+        CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "-s -X github.com/42wim/matterbridge/version.GitHash=$(git log --pretty=format:'%h' -n 1)" -o output/lin/matterbridge-$VERSION-linux-amd64
-        GOOS=windows GOARCH=amd64 go build -mod=vendor -ldflags "-s -X main.githash=$(git log --pretty=format:'%h' -n 1)" -o output/win/matterbridge-$VERSION-windows-amd64.exe
+        CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -ldflags "-s -X github.com/42wim/matterbridge/version.GitHash=$(git log --pretty=format:'%h' -n 1)" -o output/win/matterbridge-$VERSION-windows-amd64.exe
-        GOOS=darwin GOARCH=amd64 go build -mod=vendor -ldflags "-s -X main.githash=$(git log --pretty=format:'%h' -n 1)" -o output/mac/matterbridge-$VERSION-darwin-amd64
+        CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -ldflags "-s -X github.com/42wim/matterbridge/version.GitHash=$(git log --pretty=format:'%h' -n 1)" -o output/mac/matterbridge-$VERSION-darwin-amd64
     - name: Upload linux 64-bit
-      if: startsWith(matrix.go-version,'1.14')
+      if: startsWith(matrix.go-version,'1.17')
       uses: actions/upload-artifact@v2
       with:
         name: matterbridge-linux-64bit
         path: output/lin
     - name: Upload windows 64-bit
-      if: startsWith(matrix.go-version,'1.14')
+      if: startsWith(matrix.go-version,'1.17')
       uses: actions/upload-artifact@v2
       with:
         name: matterbridge-windows-64bit
         path: output/win
     - name: Upload darwin 64-bit
-      if: startsWith(matrix.go-version,'1.14')
+      if: startsWith(matrix.go-version,'1.17')
       uses: actions/upload-artifact@v2
       with:
         name: matterbridge-darwin-64bit

.github/workflows/docker.yml

@@ -0,0 +1,68 @@
+name: docker
+
+on:
+  push:
+    branches:
+      - 'master'
+    tags:
+      - 'v*'
+  pull_request:
+    branches:
+      - 'master'
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+        with:
+          platforms: amd64,arm64
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: 42wim/matterbridge,ghcr.io/42wim/matterbridge
+          flavor: |
+            latest=true
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern=stable
+            type=semver,pattern={{major}}
+            type=semver,pattern={{major}}.{{minor}}
+      -
+        name: Login to DockerHub
+        uses: docker/login-action@v1 
+        if: github.event_name != 'pull_request'
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      -
+        name: Log into registry ghcr.io
+        uses: docker/login-action@v1
+        if: github.event_name != 'pull_request'
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      -
+        name: Build and push
+        id: docker_build
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+

.golangci.yaml

@@ -7,7 +7,7 @@ run:
   # concurrency: 4
 
   # timeout for analysis, e.g. 30s, 5m, default is 1m
-  deadline: 2m
+  deadline: 5m
 
   # exit code when at least one issue was found, default is 1
   issues-exit-code: 1
@@ -91,7 +91,6 @@ linters-settings:
     # Correct spellings using locale preferences for US or UK.
     # Default is to use a neutral variety of English.
     # Setting locale to US will correct the British spelling of 'colour' to 'color'.
-    locale: US
   lll:
     # max line length, lines longer will be reported. Default is 120.
     # '\t' is counted as 1 character by default, and can be changed with the tab-width option
@@ -180,7 +179,15 @@ linters:
     - goerr113
     - testpackage
     - godot
-
+    - interfacer
+    - goheader
+    - noctx
+    - gci
+    - errorlint
+    - nlreturn
+    - exhaustivestruct
+    - forbidigo
+    - wrapcheck
 
 # rules to deal with reported isues
 issues:

.goreleaser.yml

@@ -18,8 +18,11 @@ builds:
     - arm
     - arm64
     - 386
+  goarm:
+    - 6
+    - 7
   ldflags:
-    - -s -w -X main.githash={{.ShortCommit}}
+    - -s -w -X github.com/42wim/matterbridge/version.GitHash={{.ShortCommit}}
 
 archives:
   -

Dockerfile

@@ -1,13 +1,11 @@
 FROM alpine:edge AS builder
 
-COPY . /go/src/github.com/42wim/matterbridge
+COPY . /go/src/matterbridge
-RUN apk update && apk add go git gcc musl-dev \
+RUN apk --no-cache add go git \
-        && cd /go/src/github.com/42wim/matterbridge \
+        && cd /go/src/matterbridge \
-        && export GOPATH=/go \
+        && CGO_ENABLED=0 go build -mod vendor -ldflags "-X github.com/42wim/matterbridge/version.GitHash=$(git log --pretty=format:'%h' -n 1)" -o /bin/matterbridge
-        && go get \
-        && go build -x -ldflags "-X main.githash=$(git log --pretty=format:'%h' -n 1)" -o /bin/matterbridge
 
-FROM alpine:edge
+FROM alpine
 RUN apk --no-cache add ca-certificates mailcap
 COPY --from=builder /bin/matterbridge /bin/matterbridge
 RUN mkdir /etc/matterbridge \

README.md

@@ -67,10 +67,11 @@ And more...
       - [Bridge slack (#general) - discord (general)](#bridge-slack-general---discord-general)
   - [Running](#running)
     - [Docker](#docker)
+    - [Systemd](#systemd)
   - [Changelog](#changelog)
   - [FAQ](#faq)
   - [Related projects](#related-projects)
-  - [Articles](#articles)
+  - [Articles / Tutorials](#articles--tutorials)
   - [Thanks](#thanks)
 
 ## Features
@@ -91,15 +92,18 @@ And more...
 - [IRC](http://www.mirc.com/servers.html)
 - [Keybase](https://keybase.io)
 - [Matrix](https://matrix.org)
-- [Mattermost](https://github.com/mattermost/mattermost-server/) 4.x, 5.x
+- [Mattermost](https://github.com/mattermost/mattermost-server/)
 - [Microsoft Teams](https://teams.microsoft.com)
+- [Mumble](https://www.mumble.info/)
 - [Nextcloud Talk](https://nextcloud.com/talk/)
 - [Rocket.chat](https://rocket.chat)
 - [Slack](https://slack.com)
 - [Ssh-chat](https://github.com/shazow/ssh-chat)
-- [Steam](https://store.steampowered.com/)
+- ~~[Steam](https://store.steampowered.com/)~~
+  - Not supported anymore, see [here](https://github.com/Philipp15b/go-steam/issues/94) for more info.
 - [Telegram](https://telegram.org)
 - [Twitch](https://twitch.tv)
+- [VK](https://vk.com/)
 - [WhatsApp](https://www.whatsapp.com/)
 - [XMPP](https://xmpp.org)
 - [Zulip](https://zulipchat.com)
@@ -107,11 +111,15 @@ And more...
 ### 3rd party via matterbridge api
 
 - [Discourse](https://github.com/DeclanHoare/matterbabble)
+- [Facebook messenger](https://github.com/powerjungle/fbridge-asyncio)
 - [Facebook messenger](https://github.com/VictorNine/fbridge)
 - [Minecraft](https://github.com/elytra/MatterLink)
+- [Minecraft](https://github.com/raws/mattercraft)
+- [Minecraft](https://gitlab.com/Programie/MatterBukkit)
 - [Reddit](https://github.com/bonehurtingjuice/mattereddit)
 - [Counter-Strike, half-life and more](https://forums.alliedmods.net/showthread.php?t=319430)
 - [MatterAMXX](https://github.com/GabeIggy/MatterAMXX)
+- [Vintage Story](https://github.com/NikkyAI/vs-matterbridge)
 
 ### API
 
@@ -120,12 +128,16 @@ More info and examples on the [wiki](https://github.com/42wim/matterbridge/wiki/
 
 Used by the projects below. Feel free to make a PR to add your project to this list.
 
-- [MatterLink](https://github.com/elytra/MatterLink) (Matterbridge link for Minecraft Server chat)
+- [MatterLink](https://github.com/elytra/MatterLink) (Matterbridge link for Minecraft Forge server chat, archived)
+- [MatterCraft](https://github.com/raws/mattercraft) (Matterbridge link for Minecraft Forge server chat)
+- [MatterBukkit](https://gitlab.com/Programie/MatterBukkit) (Matterbridge link for Minecraft Bukkit/Spigot server chat)
 - [pyCord](https://github.com/NikkyAI/pyCord) (crossplatform chatbot)
 - [Mattereddit](https://github.com/bonehurtingjuice/mattereddit) (Reddit chat support)
+- [fbridge-asyncio](https://github.com/powerjungle/fbridge-asyncio) (Facebook messenger support)
 - [fbridge](https://github.com/VictorNine/fbridge) (Facebook messenger support)
 - [matterbabble](https://github.com/DeclanHoare/matterbabble) (Discourse support)
 - [MatterAMXX](https://forums.alliedmods.net/showthread.php?t=319430) (Counter-Strike, half-life and more via AMXX mod)
+- [Vintage Story](https://github.com/NikkyAI/vs-matterbridge)
 
 ## Chat with us
 
@@ -146,37 +158,45 @@ Questions or want to test on your favorite platform? Join below:
 
 ## Screenshots
 
-See https://github.com/42wim/matterbridge/wiki
+See <https://github.com/42wim/matterbridge/wiki>
 
 ## Installing / upgrading
 
 ### Binaries
 
-- Latest stable release [v1.18.0](https://github.com/42wim/matterbridge/releases/latest)
+- Latest stable release [v1.23.2](https://github.com/42wim/matterbridge/releases/latest)
 - Development releases (follows master) can be downloaded [here](https://github.com/42wim/matterbridge/actions) selecting the latest green build and then artifacts.
 
-To install or upgrade just download the latest [binary](https://github.com/42wim/matterbridge/releases/latest) and follow the instructions on the [howto](https://github.com/42wim/matterbridge/wiki/How-to-create-your-config) for a step by step walkthrough for creating your configuration.
+To install or upgrade just download the latest [binary](https://github.com/42wim/matterbridge/releases/latest). On \*nix platforms you may need to make the binary executable - you can do this by running `chmod a+x` on the binary (example: `chmod a+x matterbridge-1.20.0-linux-64bit`). After downloading (and making the binary executable, if necessary), follow the instructions on the [howto](https://github.com/42wim/matterbridge/wiki/How-to-create-your-config) for a step by step walkthrough for creating your configuration.
 
 ### Packages
 
 - [Overview](https://repology.org/metapackage/matterbridge/versions)
 - [snap](https://snapcraft.io/matterbridge)
+- [scoop](https://github.com/42wim/scoop-bucket)
 
 ## Building
 
 Most people just want to use binaries, you can find those [here](https://github.com/42wim/matterbridge/releases/latest)
 
 If you really want to build from source, follow these instructions:
-Go 1.12+ is required. Make sure you have [Go](https://golang.org/doc/install) properly installed.
+Go 1.17+ is required. Make sure you have [Go](https://golang.org/doc/install) properly installed.
 
+To install the latest stable run:
 
+```bash
+go install github.com/42wim/matterbridge
 ```
-go get github.com/42wim/matterbridge
+
+To install the latest dev run:
+
+```bash
+go install github.com/42wim/matterbridge@master
 ```
 
 You should now have matterbridge binary in the ~/go/bin directory:
 
-```
+```bash
 $ ls ~/go/bin/
 matterbridge
 ```
@@ -201,8 +221,8 @@ All possible [settings](https://github.com/42wim/matterbridge/wiki/Settings) for
 
 ```toml
 [irc]
-    [irc.freenode]
+    [irc.libera]
-    Server="irc.freenode.net:6667"
+    Server="irc.libera.chat:6667"
     Nick="yourbotname"
 
 [mattermost]
@@ -218,7 +238,7 @@ All possible [settings](https://github.com/42wim/matterbridge/wiki/Settings) for
 name="mygateway"
 enable=true
     [[gateway.inout]]
-    account="irc.freenode"
+    account="irc.libera"
     channel="#testing"
 
     [[gateway.inout]]
@@ -259,7 +279,7 @@ RemoteNickFormat="[{PROTOCOL}/{BRIDGE}] <{NICK}> "
 
 See [howto](https://github.com/42wim/matterbridge/wiki/How-to-create-your-config) for a step by step walkthrough for creating your configuration.
 
-```
+```bash
 Usage of ./matterbridge:
   -conf string
         config file (default "matterbridge.toml")
@@ -275,6 +295,10 @@ Usage of ./matterbridge:
 
 Please take a look at the [Docker Wiki page](https://github.com/42wim/matterbridge/wiki/Deploy:-Docker) for more information.
 
+### Systemd
+
+Please take a look at the [Service Files page](https://github.com/42wim/matterbridge/wiki/Service-files) for more information.
+
 ## Changelog
 
 See [changelog.md](https://github.com/42wim/matterbridge/blob/master/changelog.md)
@@ -296,19 +320,24 @@ See [FAQ](https://github.com/42wim/matterbridge/wiki/FAQ)
 - [fbridge](https://github.com/VictorNine/fbridge) (Facebook messenger support)
 - [isla](https://github.com/alphachung/isla) (Bot for Discord-Telegram groups used alongside matterbridge)
 - [matterbabble](https://github.com/DeclanHoare/matterbabble) (Connect Discourse threads to Matterbridge)
+- [nextcloud talk](https://github.com/nextcloud/talk_matterbridge) (Integrates matterbridge in Nextcloud Talk)
+- [mattercraft](https://github.com/raws/mattercraft) (Minecraft bridge)
+- [vs-matterbridge](https://github.com/NikkyAI/vs-matterbridge) (Vintage Story bridge)
 
-## Articles
+## Articles / Tutorials
 
 - [matterbridge on kubernetes](https://medium.freecodecamp.org/using-kubernetes-to-deploy-a-chat-gateway-or-when-technology-works-like-its-supposed-to-a169a8cd69a3)
-- https://mattermost.com/blog/connect-irc-to-mattermost/
+- <https://mattermost.com/blog/connect-irc-to-mattermost/>
-- https://blog.valvin.fr/2016/09/17/mattermost-et-un-channel-irc-cest-possible/
+- <https://blog.valvin.fr/2016/09/17/mattermost-et-un-channel-irc-cest-possible/>
-- https://blog.brightscout.com/top-10-mattermost-integrations/
+- <https://blog.brightscout.com/top-10-mattermost-integrations/>
-- http://bencey.co.nz/2018/09/17/bridge/
+- <https://www.algoo.fr/blog/2018/01/19/recouvrez-votre-liberte-en-quittant-slack-pour-un-mattermost-auto-heberge/>
-- https://www.algoo.fr/blog/2018/01/19/recouvrez-votre-liberte-en-quittant-slack-pour-un-mattermost-auto-heberge/
+- <https://kopano.com/blog/matterbridge-bridging-mattermost-chat/>
-- https://kopano.com/blog/matterbridge-bridging-mattermost-chat/
+- <https://www.stitcher.com/s/?eid=52382713>
-- https://www.stitcher.com/s/?eid=52382713
+- <https://daniele.tech/2019/02/how-to-use-matterbridge-to-connect-2-different-slack-workspaces/>
-- https://daniele.tech/2019/02/how-to-use-matterbridge-to-connect-2-different-slack-workspaces/
+- <https://userlinux.net/mattermost-and-matterbridge.html>
-- https://userlinux.net/mattermost-and-matterbridge.html
+- <https://nextcloud.com/blog/bridging-chat-services-in-talk/>
+- <https://minecraftchest1.wordpress.com/2021/06/05/how-to-install-and-setup-matterbridge/>
+- Youtube: [whatsapp - telegram bridging](https://www.youtube.com/watch?v=W-VXISoKtNc)
 
 ## Thanks
 
@@ -321,36 +350,40 @@ See [FAQ](https://github.com/42wim/matterbridge/wiki/FAQ)
 
 Matterbridge wouldn't exist without these libraries:
 
-- discord - https://github.com/bwmarrin/discordgo
+- discord - <https://github.com/bwmarrin/discordgo>
-- echo - https://github.com/labstack/echo
+- echo - <https://github.com/labstack/echo>
-- gitter - https://github.com/sromku/go-gitter
+- gitter - <https://github.com/sromku/go-gitter>
-- gops - https://github.com/google/gops
+- gops - <https://github.com/google/gops>
-- gozulipbot - https://github.com/ifo/gozulipbot
+- gozulipbot - <https://github.com/ifo/gozulipbot>
-- irc - https://github.com/lrstanley/girc
+- gumble - <https://github.com/layeh/gumble>
-- keybase - https://github.com/keybase/go-keybase-chat-bot
+- irc - <https://github.com/lrstanley/girc>
-- matrix - https://github.com/matrix-org/gomatrix
+- keybase - <https://github.com/keybase/go-keybase-chat-bot>
-- mattermost - https://github.com/mattermost/mattermost-server
+- matrix - <https://github.com/matrix-org/gomatrix>
-- msgraph.go - https://github.com/yaegashi/msgraph.go
+- mattermost - <https://github.com/mattermost/mattermost-server>
-- slack - https://github.com/nlopes/slack
+- msgraph.go - <https://github.com/yaegashi/msgraph.go>
-- sshchat - https://github.com/shazow/ssh-chat
+- mumble - <https://github.com/layeh/gumble>
-- steam - https://github.com/Philipp15b/go-steam
+- nctalk - <https://github.com/gary-kim/go-nc-talk>
-- telegram - https://github.com/go-telegram-bot-api/telegram-bot-api
+- slack - <https://github.com/nlopes/slack>
-- tengo - https://github.com/d5/tengo
+- sshchat - <https://github.com/shazow/ssh-chat>
-- whatsapp - https://github.com/Rhymen/go-whatsapp/
+- steam - <https://github.com/Philipp15b/go-steam>
-- xmpp - https://github.com/mattn/go-xmpp
+- telegram - <https://github.com/go-telegram-bot-api/telegram-bot-api>
-- zulip - https://github.com/ifo/gozulipbot
+- tengo - <https://github.com/d5/tengo>
+- vk - <https://github.com/SevereCloud/vksdk>
+- whatsapp - <https://github.com/Rhymen/go-whatsapp>
+- xmpp - <https://github.com/mattn/go-xmpp>
+- zulip - <https://github.com/ifo/gozulipbot>
 
 <!-- Links -->
 
 [mb-discord]: https://discord.gg/AkKPtrQ
 [mb-gitter]: https://gitter.im/42wim/matterbridge
-[mb-irc]: https://webchat.freenode.net/?channels=matterbridgechat
+[mb-irc]: https://web.libera.chat/#matterbridge
 [mb-keybase]: https://keybase.io/team/matterbridge
 [mb-matrix]: https://riot.im/app/#/room/#matterbridge:matrix.org
 [mb-mattermost]: https://framateam.org/signup_user_complete/?id=tfqm33ggop8x3qgu4boeieta6e
 [mb-msteams]: https://teams.microsoft.com/join/hj92x75gd3y7
 [mb-rocketchat]: https://open.rocket.chat/channel/matterbridge
-[mb-slack]: https://join.slack.com/matterbridgechat/shared_invite/MjEwODMxNjU1NDMwLTE0OTk2MTU3NTMtMzZkZmRiNDZhOA
+[mb-slack]: https://join.slack.com/t/matterbridgechat/shared_invite/zt-2ourq2h2-7YvyYBq2WFGC~~zEzA68_Q
 [mb-telegram]: https://t.me/Matterbridge
 [mb-twitch]: https://www.twitch.tv/matterbridge
 [mb-whatsapp]: https://www.whatsapp.com/

bridge/api/api.go

@@ -6,9 +6,10 @@ import (
 	"sync"
 	"time"
 
+	"gopkg.in/olahol/melody.v1"
+
 	"github.com/42wim/matterbridge/bridge"
 	"github.com/42wim/matterbridge/bridge/config"
-	"github.com/gorilla/websocket"
 	"github.com/labstack/echo/v4"
 	"github.com/labstack/echo/v4/middleware"
 	ring "github.com/zfjagann/golang-ring"
@@ -18,6 +19,7 @@ type API struct {
 	Messages ring.Ring
 	sync.RWMutex
 	*bridge.Config
+	mrouter *melody.Melody
 }
 
 type Message struct {
@@ -33,6 +35,32 @@ func New(cfg *bridge.Config) bridge.Bridger {
 	e := echo.New()
 	e.HideBanner = true
 	e.HidePort = true
+
+	b.mrouter = melody.New()
+	b.mrouter.HandleMessage(func(s *melody.Session, msg []byte) {
+		message := config.Message{}
+		err := json.Unmarshal(msg, &message)
+		if err != nil {
+			b.Log.Errorf("failed to decode message from byte[] '%s'", string(msg))
+			return
+		}
+		b.handleWebsocketMessage(message, s)
+	})
+	b.mrouter.HandleConnect(func(session *melody.Session) {
+		greet := b.getGreeting()
+		data, err := json.Marshal(greet)
+		if err != nil {
+			b.Log.Errorf("failed to encode message '%v'", greet)
+			return
+		}
+		err = session.Write(data)
+		if err != nil {
+			b.Log.Errorf("failed to write message '%s'", string(data))
+			return
+		}
+		// TODO: send message history buffer from `b.Messages` here
+	})
+
 	b.Messages = ring.Ring{}
 	if b.GetInt("Buffer") != 0 {
 		b.Messages.SetCapacity(b.GetInt("Buffer"))
@@ -67,13 +95,13 @@ func New(cfg *bridge.Config) bridge.Bridger {
 func (b *API) Connect() error {
 	return nil
 }
+
 func (b *API) Disconnect() error {
 	return nil
-
 }
+
 func (b *API) JoinChannel(channel config.ChannelInfo) error {
 	return nil
-
 }
 
 func (b *API) Send(msg config.Message) (string, error) {
@@ -83,7 +111,14 @@ func (b *API) Send(msg config.Message) (string, error) {
 	if msg.Event == config.EventMsgDelete {
 		return "", nil
 	}
-	b.Messages.Enqueue(&msg)
+	b.Log.Debugf("enqueueing message from %s on ring buffer", msg.Username)
+	b.Messages.Enqueue(msg)
+
+	data, err := json.Marshal(msg)
+	if err != nil {
+		b.Log.Errorf("failed to encode message  '%s'", msg)
+	}
+	_ = b.mrouter.Broadcast(data)
 	return "", nil
 }
 
@@ -131,6 +166,7 @@ func (b *API) handleStream(c echo.Context) error {
 	}
 	c.Response().Flush()
 	for {
+		// TODO: this causes issues, messages should be broadcasted to all connected clients
 		msg := b.Messages.Dequeue()
 		if msg != nil {
 			if err := json.NewEncoder(c.Response()).Encode(msg); err != nil {
@@ -142,51 +178,30 @@ func (b *API) handleStream(c echo.Context) error {
 	}
 }
 
-func (b *API) handleWebsocketMessage(message config.Message) {
+func (b *API) handleWebsocketMessage(message config.Message, s *melody.Session) {
 	message.Channel = "api"
 	message.Protocol = "api"
 	message.Account = b.Account
 	message.ID = ""
 	message.Timestamp = time.Now()
 
+	data, err := json.Marshal(message)
+	if err != nil {
+		b.Log.Errorf("failed to encode message for loopback '%v'", message)
+		return
+	}
+	_ = b.mrouter.BroadcastOthers(data, s)
+
 	b.Log.Debugf("Sending websocket message from %s on %s to gateway", message.Username, "api")
 	b.Remote <- message
 }
 
-func (b *API) writePump(conn *websocket.Conn) {
-	for {
-		msg := b.Messages.Dequeue()
-		if msg != nil {
-			err := conn.WriteJSON(msg)
-			if err != nil {
-				break
-			}
-		}
-	}
-}
-
-func (b *API) readPump(conn *websocket.Conn) {
-	for {
-		message := config.Message{}
-		err := conn.ReadJSON(&message)
-		if err != nil {
-			break
-		}
-		b.handleWebsocketMessage(message)
-	}
-}
-
 func (b *API) handleWebsocket(c echo.Context) error {
-	conn, err := websocket.Upgrade(c.Response().Writer, c.Request(), nil, 1024, 1024)
+	err := b.mrouter.HandleRequest(c.Response(), c.Request())
 	if err != nil {
+		b.Log.Errorf("error in websocket handling  '%v'", err)
 		return err
 	}
 
-	greet := b.getGreeting()
-	_ = conn.WriteJSON(greet)
-
-	go b.writePump(conn)
-	go b.readPump(conn)
-
 	return nil
 }

bridge/config/config.go

@@ -26,8 +26,11 @@ const (
 	EventAPIConnected      = "api_connected"
 	EventUserTyping        = "user_typing"
 	EventGetChannelMembers = "get_channel_members"
+	EventNoticeIRC         = "notice_irc"
 )
 
+const ParentIDNotFound = "msg-parent-not-found"
+
 type Message struct {
 	Text      string    `json:"text"`
 	Channel   string    `json:"channel"`
@@ -44,6 +47,14 @@ type Message struct {
 	Extra     map[string][]interface{}
 }
 
+func (m Message) ParentNotFound() bool {
+	return m.ParentID == ParentIDNotFound
+}
+
+func (m Message) ParentValid() bool {
+	return m.ParentID != "" && !m.ParentNotFound()
+}
+
 type FileInfo struct {
 	Name    string
 	Data    *[]byte
@@ -74,32 +85,34 @@ type ChannelMember struct {
 type ChannelMembers []ChannelMember
 
 type Protocol struct {
-	AuthCode               string // steam
+	AllowMention           []string // discord
-	BindAddress            string // mattermost, slack // DEPRECATED
+	AuthCode               string   // steam
-	Buffer                 int    // api
+	BindAddress            string   // mattermost, slack // DEPRECATED
-	Charset                string // irc
+	Buffer                 int      // api
-	ClientID               string // msteams
+	Charset                string   // irc
-	ColorNicks             bool   // only irc for now
+	ClientID               string   // msteams
-	Debug                  bool   // general
+	ColorNicks             bool     // only irc for now
-	DebugLevel             int    // only for irc now
+	Debug                  bool     // general
-	DisableWebPagePreview  bool   // telegram
+	DebugLevel             int      // only for irc now
-	EditSuffix             string // mattermost, slack, discord, telegram, gitter
+	DisableWebPagePreview  bool     // telegram
-	EditDisable            bool   // mattermost, slack, discord, telegram, gitter
+	EditSuffix             string   // mattermost, slack, discord, telegram, gitter
-	HTMLDisable            bool   // matrix
+	EditDisable            bool     // mattermost, slack, discord, telegram, gitter
-	IconURL                string // mattermost, slack
+	HTMLDisable            bool     // matrix
-	IgnoreFailureOnStart   bool   // general
+	IconURL                string   // mattermost, slack
-	IgnoreNicks            string // all protocols
+	IgnoreFailureOnStart   bool     // general
-	IgnoreMessages         string // all protocols
+	IgnoreNicks            string   // all protocols
-	Jid                    string // xmpp
+	IgnoreMessages         string   // all protocols
-	JoinDelay              string // all protocols
+	Jid                    string   // xmpp
-	Label                  string // all protocols
+	JoinDelay              string   // all protocols
-	Login                  string // mattermost, matrix
+	Label                  string   // all protocols
-	LogFile                string // general
+	Login                  string   // mattermost, matrix
+	LogFile                string   // general
 	MediaDownloadBlackList []string
 	MediaDownloadPath      string // Basically MediaServerUpload, but instead of uploading it, just write it to a file on the same server.
 	MediaDownloadSize      int    // all protocols
 	MediaServerDownload    string
 	MediaServerUpload      string
+	MediaConvertTgs        string     // telegram
 	MediaConvertWebPToPNG  bool       // telegram
 	MessageDelay           int        // IRC, time in millisecond to wait between messages
 	MessageFormat          string     // telegram
@@ -107,6 +120,7 @@ type Protocol struct {
 	MessageQueue           int        // IRC, size of message queue for flood control
 	MessageSplit           bool       // IRC, split long messages with newlines on MessageLength instead of clipping
 	Muc                    string     // xmpp
+	MxID                   string     // matrix
 	Name                   string     // all protocols
 	Nick                   string     // all protocols
 	NickFormatter          string     // mattermost, slack
@@ -116,7 +130,7 @@ type Protocol struct {
 	NicksPerRow            int        // mattermost, slack
 	NoHomeServerSuffix     bool       // matrix
 	NoSendJoinPart         bool       // all protocols
-	NoTLS                  bool       // mattermost
+	NoTLS                  bool       // mattermost, xmpp
 	Password               string     // IRC,mattermost,XMPP,matrix
 	PrefixMessagesWithNick bool       // mattemost, slack
 	PreserveThreading      bool       // slack
@@ -124,12 +138,13 @@ type Protocol struct {
 	QuoteDisable           bool       // telegram
 	QuoteFormat            string     // telegram
 	QuoteLengthLimit       int        // telegram
+	RealName               string     // IRC
 	RejoinDelay            int        // IRC
 	ReplaceMessages        [][]string // all protocols
 	ReplaceNicks           [][]string // all protocols
 	RemoteNickFormat       string     // all protocols
 	RunCommands            []string   // IRC
-	Server                 string     // IRC,mattermost,XMPP,discord
+	Server                 string     // IRC,mattermost,XMPP,discord,matrix
 	SessionFile            string     // msteams,whatsapp
 	ShowJoinPart           bool       // all protocols
 	ShowTopicChange        bool       // slack
@@ -144,7 +159,7 @@ type Protocol struct {
 	Team                   string     // mattermost, keybase
 	TeamID                 string     // msteams
 	TenantID               string     // msteams
-	Token                  string     // gitter, slack, discord, api
+	Token                  string     // gitter, slack, discord, api, matrix
 	Topic                  string     // zulip
 	URL                    string     // mattermost, slack // DEPRECATED
 	UseAPI                 bool       // mattermost, slack
@@ -153,8 +168,9 @@ type Protocol struct {
 	UseTLS                 bool       // IRC
 	UseDiscriminator       bool       // discord
 	UseFirstName           bool       // telegram
-	UseUserName            bool       // discord
+	UseUserName            bool       // discord, matrix
 	UseInsecureURL         bool       // telegram
+	UserName               string     // IRC
 	VerboseJoinPart        bool       // IRC
 	WebhookBindAddress     string     // mattermost, slack
 	WebhookURL             string     // mattermost, slack
@@ -212,6 +228,7 @@ type BridgeValues struct {
 	WhatsApp           map[string]Protocol // TODO is this struct used? Search for "SlackLegacy" for example didn't return any results
 	Zulip              map[string]Protocol
 	Keybase            map[string]Protocol
+	Mumble             map[string]Protocol
 	General            Protocol
 	Tengo              Tengo
 	Gateway            []Gateway

bridge/discord/discord.go

@@ -2,13 +2,13 @@ package bdiscord
 
 import (
 	"bytes"
-	"errors"
 	"fmt"
 	"strings"
 	"sync"
 
 	"github.com/42wim/matterbridge/bridge"
 	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/42wim/matterbridge/bridge/discord/transmitter"
 	"github.com/42wim/matterbridge/bridge/helper"
 	"github.com/matterbridge/discordgo"
 )
@@ -20,12 +20,9 @@ type Bdiscord struct {
 
 	c *discordgo.Session
 
-	nick            string
+	nick    string
-	userID          string
+	userID  string
-	guildID         string
+	guildID string
-	webhookID       string
-	webhookToken    string
-	canEditWebhooks bool
 
 	channelsMutex  sync.RWMutex
 	channels       []*discordgo.Channel
@@ -34,8 +31,10 @@ type Bdiscord struct {
 	membersMutex  sync.RWMutex
 	userMemberMap map[string]*discordgo.Member
 	nickMemberMap map[string]*discordgo.Member
-	webhookCache  map[string]string
+
-	webhookMutex  sync.RWMutex
+	// Webhook specific logic
+	useAutoWebhooks bool
+	transmitter     *transmitter.Transmitter
 }
 
 func New(cfg *bridge.Config) bridge.Bridger {
@@ -43,24 +42,18 @@ func New(cfg *bridge.Config) bridge.Bridger {
 	b.userMemberMap = make(map[string]*discordgo.Member)
 	b.nickMemberMap = make(map[string]*discordgo.Member)
 	b.channelInfoMap = make(map[string]*config.ChannelInfo)
-	b.webhookCache = make(map[string]string)
+
-	if b.GetString("WebhookURL") != "" {
+	b.useAutoWebhooks = b.GetBool("AutoWebhooks")
-		b.Log.Debug("Configuring Discord Incoming Webhook")
+	if b.useAutoWebhooks {
-		b.webhookID, b.webhookToken = b.splitURL(b.GetString("WebhookURL"))
+		b.Log.Debug("Using automatic webhooks")
 	}
 	return b
 }
 
 func (b *Bdiscord) Connect() error {
 	var err error
-	var guildFound bool
 	token := b.GetString("Token")
 	b.Log.Info("Connecting")
-	if b.GetString("WebhookURL") == "" {
-		b.Log.Info("Connecting using token")
-	} else {
-		b.Log.Info("Connecting using webhookurl (for posting) and token")
-	}
 	if !strings.HasPrefix(b.GetString("Token"), "Bot ") {
 		token = "Bot " + b.GetString("Token")
 	}
@@ -82,6 +75,11 @@ func (b *Bdiscord) Connect() error {
 	b.c.AddHandler(b.messageDeleteBulk)
 	b.c.AddHandler(b.memberAdd)
 	b.c.AddHandler(b.memberRemove)
+	// Add privileged intent for guild member tracking. This is needed to track nicks
+	// for display names and @mention translation
+	b.c.Identify.Intents = discordgo.MakeIntent(discordgo.IntentsAllWithoutPrivileged |
+		discordgo.IntentsGuildMembers)
+
 	err = b.c.Open()
 	if err != nil {
 		return err
@@ -97,62 +95,107 @@ func (b *Bdiscord) Connect() error {
 	serverName := strings.Replace(b.GetString("Server"), "ID:", "", -1)
 	b.nick = userinfo.Username
 	b.userID = userinfo.ID
+
+	// Try and find this account's guild, and populate channels
 	b.channelsMutex.Lock()
 	for _, guild := range guilds {
-		if guild.Name == serverName || guild.ID == serverName {
+		// Skip, if the server name does not match the visible name or the ID
-			b.channels, err = b.c.GuildChannels(guild.ID)
+		if guild.Name != serverName && guild.ID != serverName {
-			if err != nil {
+			continue
-				break
-			}
-			b.guildID = guild.ID
-			guildFound = true
 		}
+
+		// Complain about an ambiguous Server setting. Two Discord servers could have the same title!
+		// For IDs, practically this will never happen. It would only trigger if some server's name is also an ID.
+		if b.guildID != "" {
+			return fmt.Errorf("found multiple Discord servers with the same name %#v, expected to see only one", serverName)
+		}
+
+		// Getting this guild's channel could result in a permission error
+		b.channels, err = b.c.GuildChannels(guild.ID)
+		if err != nil {
+			return fmt.Errorf("could not get %#v's channels: %w", b.GetString("Server"), err)
+		}
+
+		b.guildID = guild.ID
 	}
 	b.channelsMutex.Unlock()
-	if !guildFound {
+
-		msg := fmt.Sprintf("Server \"%s\" not found", b.GetString("Server"))
+	// If we couldn't find a guild, we print extra debug information and return a nice error
-		err = errors.New(msg)
+	if b.guildID == "" {
-		b.Log.Error(msg)
+		err = fmt.Errorf("could not find Discord server %#v", b.GetString("Server"))
-		b.Log.Info("Possible values:")
+		b.Log.Error(err.Error())
+
+		// Print all of the possible server values
+		b.Log.Info("Possible server values:")
 		for _, guild := range guilds {
-			b.Log.Infof("Server=\"%s\" # Server name", guild.Name)
+			b.Log.Infof("\t- Server=%#v # by name", guild.Name)
-			b.Log.Infof("Server=\"%s\" # Server ID", guild.ID)
+			b.Log.Infof("\t- Server=%#v # by ID", guild.ID)
 		}
-	}
+
-	if err != nil {
+		// If there are no results, we should say that
+		if len(guilds) == 0 {
+			b.Log.Info("\t- (none found)")
+		}
+
 		return err
 	}
 
-	b.channelsMutex.RLock()
+	// Legacy note: WebhookURL used to have an actual webhook URL that we would edit,
-	if b.GetString("WebhookURL") == "" {
+	// but we stopped doing that due to Discord making rate limits more aggressive.
-		for _, channel := range b.channels {
+	//
-			b.Log.Debugf("found channel %#v", channel)
+	// Even older: the same WebhookURL used to be used by every channel, which is usually unexpected.
-		}
+	// This is no longer possible.
-	} else {
+	if b.GetString("WebhookURL") != "" {
-		manageWebhooks := discordgo.PermissionManageWebhooks
+		message := "The global WebhookURL setting has been removed. "
-		var channelsDenied []string
+		message += "You can get similar \"webhook editing\" behaviour by replacing this line with `AutoWebhooks=true`. "
-		for _, info := range b.Channels {
+		message += "If you rely on the old-OLD (non-editing) behaviour, can move the WebhookURL to specific channel sections."
-			id := b.getChannelID(info.Name) // note(qaisjp): this readlocks channelsMutex
+		b.Log.Errorln(message)
-			b.Log.Debugf("Verifying PermissionManageWebhooks for %s with ID %s", info.ID, id)
+		return fmt.Errorf("use of removed WebhookURL setting")
+	}
 
-			perms, permsErr := b.c.UserChannelPermissions(userinfo.ID, id)
+	if b.GetInt("debuglevel") > 0 {
-			if permsErr != nil {
+		b.Log.Debug("enabling even more discord debug")
-				b.Log.Warnf("Failed to check PermissionManageWebhooks in channel \"%s\": %s", info.Name, permsErr.Error())
+		b.c.Debug = true
-			} else if perms&manageWebhooks == manageWebhooks {
+	}
-				continue
+
+	// Initialise webhook management
+	b.transmitter = transmitter.New(b.c, b.guildID, "matterbridge", b.useAutoWebhooks)
+	b.transmitter.Log = b.Log
+
+	var webhookChannelIDs []string
+	for _, channel := range b.Channels {
+		channelID := b.getChannelID(channel.Name) // note(qaisjp): this readlocks channelsMutex
+
+		// If a WebhookURL was not explicitly provided for this channel,
+		// there are two options: just a regular bot message (ugly) or this is should be webhook sent
+		if channel.Options.WebhookURL == "" {
+			// If it should be webhook sent, we should enforce this via the transmitter
+			if b.useAutoWebhooks {
+				webhookChannelIDs = append(webhookChannelIDs, channelID)
 			}
-			channelsDenied = append(channelsDenied, fmt.Sprintf("%#v", info.Name))
+			continue
 		}
 
-		b.canEditWebhooks = len(channelsDenied) == 0
+		whID, whToken, ok := b.splitURL(channel.Options.WebhookURL)
-		if b.canEditWebhooks {
+		if !ok {
-			b.Log.Info("Can manage webhooks; will edit channel for global webhook on send")
+			return fmt.Errorf("failed to parse WebhookURL %#v for channel %#v", channel.Options.WebhookURL, channel.ID)
-		} else {
+		}
-			b.Log.Warn("Can't manage webhooks; won't edit channel for global webhook on send")
+
-			b.Log.Warn("Can't manage webhooks in channels: ", strings.Join(channelsDenied, ", "))
+		b.transmitter.AddWebhook(channelID, &discordgo.Webhook{
+			ID:        whID,
+			Token:     whToken,
+			GuildID:   b.guildID,
+			ChannelID: channelID,
+		})
+	}
+
+	if b.useAutoWebhooks {
+		err = b.transmitter.RefreshGuildWebhooks(webhookChannelIDs)
+		if err != nil {
+			b.Log.WithError(err).Println("transmitter could not refresh guild webhooks")
+			return err
 		}
 	}
-	b.channelsMutex.RUnlock()
 
 	// Obtaining guild members and initializing nickname mapping.
 	b.membersMutex.Lock()
@@ -191,8 +234,6 @@ func (b *Bdiscord) JoinChannel(channel config.ChannelInfo) error {
 func (b *Bdiscord) Send(msg config.Message) (string, error) {
 	b.Log.Debugf("=> Receiving %#v", msg)
 
-	origMsgID := msg.ID
-
 	channelID := b.getChannelID(msg.Channel)
 	if channelID == "" {
 		return "", fmt.Errorf("Could not find channelID for %v", msg.Channel)
@@ -211,79 +252,23 @@ func (b *Bdiscord) Send(msg config.Message) (string, error) {
 		msg.Text = "_" + msg.Text + "_"
 	}
 
-	// use initial webhook configured for the entire Discord account
+	// Handle prefix hint for unthreaded messages.
-	isGlobalWebhook := true
+	if msg.ParentNotFound() {
-	wID := b.webhookID
+		msg.ParentID = ""
-	wToken := b.webhookToken
+		msg.Text = fmt.Sprintf("[thread]: %s", msg.Text)
-
-	// check if have a channel specific webhook
-	b.channelsMutex.RLock()
-	if ci, ok := b.channelInfoMap[msg.Channel+b.Account]; ok {
-		if ci.Options.WebhookURL != "" {
-			wID, wToken = b.splitURL(ci.Options.WebhookURL)
-			isGlobalWebhook = false
-		}
 	}
-	b.channelsMutex.RUnlock()
 
 	// Use webhook to send the message
-	if wID != "" && msg.Event != config.EventMsgDelete {
+	useWebhooks := b.shouldMessageUseWebhooks(&msg)
-		// skip events
+	if useWebhooks && msg.Event != config.EventMsgDelete && msg.ParentID == "" {
-		if msg.Event != "" && msg.Event != config.EventUserAction && msg.Event != config.EventJoinLeave && msg.Event != config.EventTopicChange {
+		return b.handleEventWebhook(&msg, channelID)
-			return "", nil
-		}
-
-		// If we are editing a message, delete the old message
-		if msg.ID != "" {
-			msg.ID = b.getCacheID(msg.ID)
-			b.Log.Debugf("Deleting edited webhook message")
-			err := b.c.ChannelMessageDelete(channelID, msg.ID)
-			if err != nil {
-				b.Log.Errorf("Could not delete edited webhook message: %s", err)
-			}
-		}
-
-		b.Log.Debugf("Broadcasting using Webhook")
-
-		// skip empty messages
-		if msg.Text == "" && (msg.Extra == nil || len(msg.Extra["file"]) == 0) {
-			b.Log.Debugf("Skipping empty message %#v", msg)
-			return "", nil
-		}
-
-		msg.Text = helper.ClipMessage(msg.Text, MessageLength)
-		msg.Text = b.replaceUserMentions(msg.Text)
-		// discord username must be [0..32] max
-		if len(msg.Username) > 32 {
-			msg.Username = msg.Username[0:32]
-		}
-		// if we have a global webhook for this Discord account, and permission
-		// to modify webhooks (previously verified), then set its channel to
-		// the message channel before using it
-		// TODO: this isn't necessary if the last message from this webhook was
-		// sent to the current channel
-		if isGlobalWebhook && b.canEditWebhooks {
-			b.Log.Debugf("Setting webhook channel to \"%s\"", msg.Channel)
-			_, err := b.c.WebhookEdit(wID, "", "", channelID)
-			if err != nil {
-				b.Log.Errorf("Could not set webhook channel: %s", err)
-				return "", err
-			}
-		}
-		b.Log.Debugf("Processing webhook sending for message %#v", msg)
-		msg, err := b.webhookSend(&msg, wID, wToken)
-		if err != nil {
-			b.Log.Errorf("Could not broadcast via webook for message %#v: %s", msg, err)
-			return "", err
-		}
-		if msg == nil {
-			return "", nil
-		}
-
-		b.updateCacheID(origMsgID, msg.ID)
-		return msg.ID, nil
 	}
 
+	return b.handleEventBotUser(&msg, channelID)
+}
+
+// handleEventDirect handles events via the bot user
+func (b *Bdiscord) handleEventBotUser(msg *config.Message, channelID string) (string, error) {
 	b.Log.Debugf("Broadcasting using token (API)")
 
 	// Delete message
@@ -291,26 +276,25 @@ func (b *Bdiscord) Send(msg config.Message) (string, error) {
 		if msg.ID == "" {
 			return "", nil
 		}
-		msg.ID = b.getCacheID(msg.ID)
 		err := b.c.ChannelMessageDelete(channelID, msg.ID)
 		return "", err
 	}
 
 	// Upload a file if it exists
 	if msg.Extra != nil {
-		for _, rmsg := range helper.HandleExtra(&msg, b.General) {
+		for _, rmsg := range helper.HandleExtra(msg, b.General) {
-			rmsg.Text = helper.ClipMessage(rmsg.Text, MessageLength)
+			rmsg.Text = helper.ClipMessage(rmsg.Text, MessageLength, b.GetString("MessageClipped"))
 			if _, err := b.c.ChannelMessageSend(channelID, rmsg.Username+rmsg.Text); err != nil {
 				b.Log.Errorf("Could not send message %#v: %s", rmsg, err)
 			}
 		}
 		// check if we have files to upload (from slack, telegram or mattermost)
 		if len(msg.Extra["file"]) > 0 {
-			return b.handleUploadFile(&msg, channelID)
+			return b.handleUploadFile(msg, channelID)
 		}
 	}
 
-	msg.Text = helper.ClipMessage(msg.Text, MessageLength)
+	msg.Text = helper.ClipMessage(msg.Text, MessageLength, b.GetString("MessageClipped"))
 	msg.Text = b.replaceUserMentions(msg.Text)
 
 	// Edit message
@@ -319,54 +303,28 @@ func (b *Bdiscord) Send(msg config.Message) (string, error) {
 		return msg.ID, err
 	}
 
+	m := discordgo.MessageSend{
+		Content:         msg.Username + msg.Text,
+		AllowedMentions: b.getAllowedMentions(),
+	}
+
+	if msg.ParentValid() {
+		m.Reference = &discordgo.MessageReference{
+			MessageID: msg.ParentID,
+			ChannelID: channelID,
+			GuildID:   b.guildID,
+		}
+	}
+
 	// Post normal message
-	res, err := b.c.ChannelMessageSend(channelID, msg.Username+msg.Text)
+	res, err := b.c.ChannelMessageSendComplex(channelID, &m)
 	if err != nil {
 		return "", err
 	}
+
 	return res.ID, nil
 }
 
-// useWebhook returns true if we have a webhook defined somewhere
-func (b *Bdiscord) useWebhook() bool {
-	if b.GetString("WebhookURL") != "" {
-		return true
-	}
-
-	b.channelsMutex.RLock()
-	defer b.channelsMutex.RUnlock()
-
-	for _, channel := range b.channelInfoMap {
-		if channel.Options.WebhookURL != "" {
-			return true
-		}
-	}
-	return false
-}
-
-// isWebhookID returns true if the specified id is used in a defined webhook
-func (b *Bdiscord) isWebhookID(id string) bool {
-	if b.GetString("WebhookURL") != "" {
-		wID, _ := b.splitURL(b.GetString("WebhookURL"))
-		if wID == id {
-			return true
-		}
-	}
-
-	b.channelsMutex.RLock()
-	defer b.channelsMutex.RUnlock()
-
-	for _, channel := range b.channelInfoMap {
-		if channel.Options.WebhookURL != "" {
-			wID, _ := b.splitURL(channel.Options.WebhookURL)
-			if wID == id {
-				return true
-			}
-		}
-	}
-	return false
-}
-
 // handleUploadFile handles native upload of files
 func (b *Bdiscord) handleUploadFile(msg *config.Message, channelID string) (string, error) {
 	var err error
@@ -378,8 +336,9 @@ func (b *Bdiscord) handleUploadFile(msg *config.Message, channelID string) (stri
 			Reader:      bytes.NewReader(*fi.Data),
 		}
 		m := discordgo.MessageSend{
-			Content: msg.Username + fi.Comment,
+			Content:         msg.Username + fi.Comment,
-			Files:   []*discordgo.File{&file},
+			Files:           []*discordgo.File{&file},
+			AllowedMentions: b.getAllowedMentions(),
 		}
 		_, err = b.c.ChannelMessageSendComplex(channelID, &m)
 		if err != nil {
@@ -388,83 +347,3 @@ func (b *Bdiscord) handleUploadFile(msg *config.Message, channelID string) (stri
 	}
 	return "", nil
 }
-
-// webhookSend send one or more message via webhook, taking care of file
-// uploads (from slack, telegram or mattermost).
-// Returns messageID and error.
-func (b *Bdiscord) webhookSend(msg *config.Message, webhookID, token string) (*discordgo.Message, error) {
-	var (
-		res *discordgo.Message
-		err error
-	)
-
-	// If avatar is unset, check if UseLocalAvatar contains the message's
-	// account or protocol, and if so, try to find a local avatar
-	if msg.Avatar == "" {
-		for _, val := range b.GetStringSlice("UseLocalAvatar") {
-			if msg.Protocol == val || msg.Account == val {
-				if avatar := b.findAvatar(msg); avatar != "" {
-					msg.Avatar = avatar
-				}
-				break
-			}
-		}
-	}
-
-	// WebhookParams can have either `Content` or `File`.
-
-	// We can't send empty messages.
-	if msg.Text != "" {
-		res, err = b.c.WebhookExecute(
-			webhookID,
-			token,
-			true,
-			&discordgo.WebhookParams{
-				Content:   msg.Text,
-				Username:  msg.Username,
-				AvatarURL: msg.Avatar,
-			},
-		)
-		if err != nil {
-			b.Log.Errorf("Could not send text (%s) for message %#v: %s", msg.Text, msg, err)
-		}
-	}
-
-	if msg.Extra != nil {
-		for _, f := range msg.Extra["file"] {
-			fi := f.(config.FileInfo)
-			file := discordgo.File{
-				Name:        fi.Name,
-				ContentType: "",
-				Reader:      bytes.NewReader(*fi.Data),
-			}
-			content := ""
-			if msg.Text == "" {
-				content = fi.Comment
-			}
-			_, e2 := b.c.WebhookExecute(
-				webhookID,
-				token,
-				false,
-				&discordgo.WebhookParams{
-					Username:  msg.Username,
-					AvatarURL: msg.Avatar,
-					File:      &file,
-					Content:   content,
-				},
-			)
-			if e2 != nil {
-				b.Log.Errorf("Could not send file %#v for message %#v: %s", file, msg, e2)
-			}
-		}
-	}
-	return res, err
-}
-
-func (b *Bdiscord) findAvatar(m *config.Message) string {
-	member, err := b.getGuildMemberByNick(m.Username)
-	if err != nil {
-		return ""
-	}
-	return member.User.AvatarURL("")
-}

bridge/discord/handlers.go

@@ -69,7 +69,7 @@ func (b *Bdiscord) messageCreate(s *discordgo.Session, m *discordgo.MessageCreat
 		return
 	}
 	// if using webhooks, do not relay if it's ours
-	if b.useWebhook() && m.Author.Bot && b.isWebhookID(m.Author.ID) {
+	if m.Author.Bot && b.transmitter.HasWebhook(m.Author.ID) {
 		return
 	}
 
@@ -127,6 +127,11 @@ func (b *Bdiscord) messageCreate(s *discordgo.Session, m *discordgo.MessageCreat
 	// Replace emotes
 	rmsg.Text = replaceEmotes(rmsg.Text)
 
+	// Add our parent id if it exists, and if it's not referring to a message in another channel
+	if ref := m.MessageReference; ref != nil && ref.ChannelID == m.ChannelID {
+		rmsg.ParentID = ref.MessageID
+	}
+
 	b.Log.Debugf("<= Sending message from %s on %s to gateway", m.Author.Username, b.Account)
 	b.Log.Debugf("<= Message is %#v", rmsg)
 	b.Remote <- rmsg

bridge/discord/helpers.go

@@ -9,6 +9,30 @@ import (
 	"github.com/matterbridge/discordgo"
 )
 
+func (b *Bdiscord) getAllowedMentions() *discordgo.MessageAllowedMentions {
+	// If AllowMention is not specified, then allow all mentions (default Discord behavior)
+	if !b.IsKeySet("AllowMention") {
+		return nil
+	}
+
+	// Otherwise, allow only the mentions that are specified
+	allowedMentionTypes := make([]discordgo.AllowedMentionType, 0, 3)
+	for _, m := range b.GetStringSlice("AllowMention") {
+		switch m {
+		case "everyone":
+			allowedMentionTypes = append(allowedMentionTypes, discordgo.AllowedMentionTypeEveryone)
+		case "roles":
+			allowedMentionTypes = append(allowedMentionTypes, discordgo.AllowedMentionTypeRoles)
+		case "users":
+			allowedMentionTypes = append(allowedMentionTypes, discordgo.AllowedMentionTypeUsers)
+		}
+	}
+
+	return &discordgo.MessageAllowedMentions{
+		Parse: allowedMentionTypes,
+	}
+}
+
 func (b *Bdiscord) getNick(user *discordgo.User, guildID string) string {
 	b.membersMutex.RLock()
 	defer b.membersMutex.RUnlock()
@@ -196,7 +220,7 @@ func (b *Bdiscord) replaceAction(text string) (string, bool) {
 }
 
 // splitURL splits a webhookURL and returns the ID and token.
-func (b *Bdiscord) splitURL(url string) (string, string) {
+func (b *Bdiscord) splitURL(url string) (string, string, bool) {
 	const (
 		expectedWebhookSplitCount = 7
 		webhookIdxID              = 5
@@ -204,43 +228,9 @@ func (b *Bdiscord) splitURL(url string) (string, string) {
 	)
 	webhookURLSplit := strings.Split(url, "/")
 	if len(webhookURLSplit) != expectedWebhookSplitCount {
-		b.Log.Fatalf("%s is no correct discord WebhookURL", url)
+		return "", "", false
 	}
-	return webhookURLSplit[webhookIdxID], webhookURLSplit[webhookIdxToken]
+	return webhookURLSplit[webhookIdxID], webhookURLSplit[webhookIdxToken], true
-}
-
-// getcacheID tries to find a corresponding msgID in the webhook cache.
-// if not found returns the original request.
-func (b *Bdiscord) getCacheID(msgID string) string {
-	b.webhookMutex.RLock()
-	defer b.webhookMutex.RUnlock()
-	for k, v := range b.webhookCache {
-		if msgID == k {
-			return v
-		}
-	}
-	return msgID
-}
-
-// updateCacheID updates the cache so that the newID takes the place of
-// the original ID. This is used for edit/deletes in combination with webhooks
-// as editing a message via webhook means deleting the message and creating a
-// new message (with a new ID). This ID needs to be set instead of the original ID
-func (b *Bdiscord) updateCacheID(origID, newID string) {
-	b.webhookMutex.Lock()
-	match := false
-	for k, v := range b.webhookCache {
-		if v == origID {
-			delete(b.webhookCache, k)
-			b.webhookCache[origID] = newID
-			match = true
-			continue
-		}
-	}
-	if !match && origID != "" {
-		b.webhookCache[origID] = newID
-	}
-	b.webhookMutex.Unlock()
 }
 
 func enumerateUsernames(s string) []string {

bridge/discord/transmitter/transmitter.go

@@ -0,0 +1,257 @@
+// Package transmitter provides functionality for transmitting
+// arbitrary webhook messages to Discord.
+//
+// The package provides the following functionality:
+//
+// - Creating new webhooks, whenever necessary
+// - Loading webhooks that we have previously created
+// - Sending new messages
+// - Editing messages, via message ID
+// - Deleting messages, via message ID
+//
+// The package has been designed for matterbridge, but with other
+// Go bots in mind. The public API should be matterbridge-agnostic.
+package transmitter
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/matterbridge/discordgo"
+	log "github.com/sirupsen/logrus"
+)
+
+// A Transmitter represents a message manager for a single guild.
+type Transmitter struct {
+	session    *discordgo.Session
+	guild      string
+	title      string
+	autoCreate bool
+
+	// channelWebhooks maps from a channel ID to a webhook instance
+	channelWebhooks map[string]*discordgo.Webhook
+
+	mutex sync.RWMutex
+
+	Log *log.Entry
+}
+
+// ErrWebhookNotFound is returned when a valid webhook for this channel/message combination does not exist
+var ErrWebhookNotFound = errors.New("webhook for this channel and message does not exist")
+
+// ErrPermissionDenied is returned if the bot does not have permission to manage webhooks.
+//
+// Bots can be granted a guild-wide permission and channel-specific permissions to manage webhooks.
+// Despite potentially having guild-wide permission, channel specific overrides could deny a bot's permission to manage webhooks.
+var ErrPermissionDenied = errors.New("missing 'Manage Webhooks' permission")
+
+// New returns a new Transmitter given a Discord session, guild ID, and title.
+func New(session *discordgo.Session, guild string, title string, autoCreate bool) *Transmitter {
+	return &Transmitter{
+		session:    session,
+		guild:      guild,
+		title:      title,
+		autoCreate: autoCreate,
+
+		channelWebhooks: make(map[string]*discordgo.Webhook),
+
+		Log: log.NewEntry(log.StandardLogger()),
+	}
+}
+
+// Send transmits a message to the given channel with the provided webhook data, and waits until Discord responds with message data.
+func (t *Transmitter) Send(channelID string, params *discordgo.WebhookParams) (*discordgo.Message, error) {
+	wh, err := t.getOrCreateWebhook(channelID)
+	if err != nil {
+		return nil, err
+	}
+
+	msg, err := t.session.WebhookExecute(wh.ID, wh.Token, true, params)
+	if err != nil {
+		return nil, fmt.Errorf("execute failed: %w", err)
+	}
+
+	return msg, nil
+}
+
+// Edit will edit a message in a channel, if possible.
+func (t *Transmitter) Edit(channelID string, messageID string, params *discordgo.WebhookParams) error {
+	wh := t.getWebhook(channelID)
+
+	if wh == nil {
+		return ErrWebhookNotFound
+	}
+
+	uri := discordgo.EndpointWebhookToken(wh.ID, wh.Token) + "/messages/" + messageID
+	_, err := t.session.RequestWithBucketID("PATCH", uri, params, discordgo.EndpointWebhookToken("", ""))
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// HasWebhook checks whether the transmitter is using a particular webhook.
+func (t *Transmitter) HasWebhook(id string) bool {
+	t.mutex.RLock()
+	defer t.mutex.RUnlock()
+
+	for _, wh := range t.channelWebhooks {
+		if wh.ID == id {
+			return true
+		}
+	}
+
+	return false
+}
+
+// AddWebhook allows you to register a channel's webhook with the transmitter.
+func (t *Transmitter) AddWebhook(channelID string, webhook *discordgo.Webhook) bool {
+	t.Log.Debugf("Manually added webhook %#v to channel %#v", webhook.ID, channelID)
+	t.mutex.Lock()
+	defer t.mutex.Unlock()
+
+	_, replaced := t.channelWebhooks[channelID]
+	t.channelWebhooks[channelID] = webhook
+	return replaced
+}
+
+// RefreshGuildWebhooks loads "relevant" webhooks into the transmitter, with careful permission handling.
+//
+// Notes:
+//
+// - A webhook is "relevant" if it was created by this bot -- the ApplicationID should match the bot's ID.
+// - The term "having permission" means having the "Manage Webhooks" permission. See ErrPermissionDenied for more information.
+// - This function is additive and will not unload previously loaded webhooks.
+// - A nil channelIDs slice is treated the same as an empty one.
+//
+// If the bot has guild-wide permission:
+//
+// 1. it will load any "relevant" webhooks from the entire guild
+// 2. the given slice is ignored
+//
+// If the bot does not have guild-wide permission:
+//
+// 1. it will load any "relevant" webhooks in each channel
+// 2. a single error will be returned if any error occurs (incl. if there is no permission for any of these channels)
+//
+// If any channel has more than one "relevant" webhook, it will randomly pick one.
+func (t *Transmitter) RefreshGuildWebhooks(channelIDs []string) error {
+	t.Log.Debugln("Refreshing guild webhooks")
+
+	botID, err := getDiscordUserID(t.session)
+	if err != nil {
+		return fmt.Errorf("could not get current user: %w", err)
+	}
+
+	// Get all existing webhooks
+	hooks, err := t.session.GuildWebhooks(t.guild)
+	if err != nil {
+		switch {
+		case isDiscordPermissionError(err):
+			// We fallback on manually fetching hooks from individual channels
+			// if we don't have the "Manage Webhooks" permission globally.
+			// We can only do this if we were provided channelIDs, though.
+			if len(channelIDs) == 0 {
+				return ErrPermissionDenied
+			}
+			t.Log.Debugln("Missing global 'Manage Webhooks' permission, falling back on per-channel permission")
+			return t.fetchChannelsHooks(channelIDs, botID)
+		default:
+			return fmt.Errorf("could not get webhooks: %w", err)
+		}
+	}
+
+	t.Log.Debugln("Refreshing guild webhooks using global permission")
+	t.assignHooksByAppID(hooks, botID, false)
+	return nil
+}
+
+// createWebhook creates a webhook for a specific channel.
+func (t *Transmitter) createWebhook(channel string) (*discordgo.Webhook, error) {
+	t.mutex.Lock()
+	defer t.mutex.Unlock()
+
+	wh, err := t.session.WebhookCreate(channel, t.title+time.Now().Format(" 3:04:05PM"), "")
+	if err != nil {
+		return nil, err
+	}
+
+	t.channelWebhooks[channel] = wh
+	return wh, nil
+}
+
+func (t *Transmitter) getWebhook(channel string) *discordgo.Webhook {
+	t.mutex.RLock()
+	defer t.mutex.RUnlock()
+
+	return t.channelWebhooks[channel]
+}
+
+func (t *Transmitter) getOrCreateWebhook(channelID string) (*discordgo.Webhook, error) {
+	// If we have a webhook for this channel, immediately return it
+	wh := t.getWebhook(channelID)
+	if wh != nil {
+		return wh, nil
+	}
+
+	// Early exit if we don't want to automatically create one
+	if !t.autoCreate {
+		return nil, ErrWebhookNotFound
+	}
+
+	t.Log.Infof("Creating a webhook for %s\n", channelID)
+	wh, err := t.createWebhook(channelID)
+	if err != nil {
+		return nil, fmt.Errorf("could not create webhook: %w", err)
+	}
+
+	return wh, nil
+}
+
+// fetchChannelsHooks fetches hooks for the given channelIDs and calls assignHooksByAppID for each channel's hooks
+func (t *Transmitter) fetchChannelsHooks(channelIDs []string, botID string) error {
+	// For each channel, search for relevant hooks
+	var failedHooks []string
+	for _, channelID := range channelIDs {
+		hooks, err := t.session.ChannelWebhooks(channelID)
+		if err != nil {
+			failedHooks = append(failedHooks, "\n- "+channelID+": "+err.Error())
+			continue
+		}
+		t.assignHooksByAppID(hooks, botID, true)
+	}
+
+	// Compose an error if any hooks failed
+	if len(failedHooks) > 0 {
+		return errors.New("failed to fetch hooks:" + strings.Join(failedHooks, ""))
+	}
+
+	return nil
+}
+
+func (t *Transmitter) assignHooksByAppID(hooks []*discordgo.Webhook, appID string, channelTargeted bool) {
+	logLine := "Picking up webhook"
+	if channelTargeted {
+		logLine += " (channel targeted)"
+	}
+
+	t.mutex.Lock()
+	defer t.mutex.Unlock()
+
+	for _, wh := range hooks {
+		if wh.ApplicationID != appID {
+			continue
+		}
+
+		t.channelWebhooks[wh.ChannelID] = wh
+		t.Log.WithFields(log.Fields{
+			"id":      wh.ID,
+			"name":    wh.Name,
+			"channel": wh.ChannelID,
+		}).Println(logLine)
+	}
+}

bridge/discord/transmitter/utils.go

@@ -0,0 +1,32 @@
+package transmitter
+
+import (
+	"github.com/matterbridge/discordgo"
+)
+
+// isDiscordPermissionError returns false for nil, and true if a Discord RESTError with code discordgo.ErrorCodeMissionPermissions
+func isDiscordPermissionError(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	restErr, ok := err.(*discordgo.RESTError)
+	if !ok {
+		return false
+	}
+
+	return restErr.Message != nil && restErr.Message.Code == discordgo.ErrCodeMissingPermissions
+}
+
+// getDiscordUserID gets own user ID from state, and fallback on API request
+func getDiscordUserID(session *discordgo.Session) (string, error) {
+	if user := session.State.User; user != nil {
+		return user.ID, nil
+	}
+
+	user, err := session.User("@me")
+	if err != nil {
+		return "", err
+	}
+	return user.ID, nil
+}

bridge/discord/webhook.go

@@ -0,0 +1,150 @@
+package bdiscord
+
+import (
+	"bytes"
+
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/42wim/matterbridge/bridge/helper"
+	"github.com/matterbridge/discordgo"
+)
+
+// shouldMessageUseWebhooks checks if have a channel specific webhook, if we're not using auto webhooks
+func (b *Bdiscord) shouldMessageUseWebhooks(msg *config.Message) bool {
+	if b.useAutoWebhooks {
+		return true
+	}
+
+	b.channelsMutex.RLock()
+	defer b.channelsMutex.RUnlock()
+	if ci, ok := b.channelInfoMap[msg.Channel+b.Account]; ok {
+		if ci.Options.WebhookURL != "" {
+			return true
+		}
+	}
+	return false
+}
+
+// maybeGetLocalAvatar checks if UseLocalAvatar contains the message's
+// account or protocol, and if so, returns the Discord avatar (if exists)
+func (b *Bdiscord) maybeGetLocalAvatar(msg *config.Message) string {
+	for _, val := range b.GetStringSlice("UseLocalAvatar") {
+		if msg.Protocol != val && msg.Account != val {
+			continue
+		}
+
+		member, err := b.getGuildMemberByNick(msg.Username)
+		if err != nil {
+			return ""
+		}
+
+		return member.User.AvatarURL("")
+	}
+	return ""
+}
+
+// webhookSend send one or more message via webhook, taking care of file
+// uploads (from slack, telegram or mattermost).
+// Returns messageID and error.
+func (b *Bdiscord) webhookSend(msg *config.Message, channelID string) (*discordgo.Message, error) {
+	var (
+		res *discordgo.Message
+		err error
+	)
+
+	// If avatar is unset, mutate the message to include the local avatar (but only if settings say we should do this)
+	if msg.Avatar == "" {
+		msg.Avatar = b.maybeGetLocalAvatar(msg)
+	}
+
+	// WebhookParams can have either `Content` or `File`.
+
+	// We can't send empty messages.
+	if msg.Text != "" {
+		res, err = b.transmitter.Send(
+			channelID,
+			&discordgo.WebhookParams{
+				Content:         msg.Text,
+				Username:        msg.Username,
+				AvatarURL:       msg.Avatar,
+				AllowedMentions: b.getAllowedMentions(),
+			},
+		)
+		if err != nil {
+			b.Log.Errorf("Could not send text (%s) for message %#v: %s", msg.Text, msg, err)
+		}
+	}
+
+	if msg.Extra != nil {
+		for _, f := range msg.Extra["file"] {
+			fi := f.(config.FileInfo)
+			file := discordgo.File{
+				Name:        fi.Name,
+				ContentType: "",
+				Reader:      bytes.NewReader(*fi.Data),
+			}
+			content := ""
+			if msg.Text == "" {
+				content = fi.Comment
+			}
+			_, e2 := b.transmitter.Send(
+				channelID,
+				&discordgo.WebhookParams{
+					Username:        msg.Username,
+					AvatarURL:       msg.Avatar,
+					File:            &file,
+					Content:         content,
+					AllowedMentions: b.getAllowedMentions(),
+				},
+			)
+			if e2 != nil {
+				b.Log.Errorf("Could not send file %#v for message %#v: %s", file, msg, e2)
+			}
+		}
+	}
+	return res, err
+}
+
+func (b *Bdiscord) handleEventWebhook(msg *config.Message, channelID string) (string, error) {
+	// skip events
+	if msg.Event != "" && msg.Event != config.EventUserAction && msg.Event != config.EventJoinLeave && msg.Event != config.EventTopicChange {
+		return "", nil
+	}
+
+	// skip empty messages
+	if msg.Text == "" && (msg.Extra == nil || len(msg.Extra["file"]) == 0) {
+		b.Log.Debugf("Skipping empty message %#v", msg)
+		return "", nil
+	}
+
+	msg.Text = helper.ClipMessage(msg.Text, MessageLength, b.GetString("MessageClipped"))
+	msg.Text = b.replaceUserMentions(msg.Text)
+	// discord username must be [0..32] max
+	if len(msg.Username) > 32 {
+		msg.Username = msg.Username[0:32]
+	}
+
+	if msg.ID != "" {
+		b.Log.Debugf("Editing webhook message")
+		err := b.transmitter.Edit(channelID, msg.ID, &discordgo.WebhookParams{
+			Content:         msg.Text,
+			Username:        msg.Username,
+			AllowedMentions: b.getAllowedMentions(),
+		})
+		if err == nil {
+			return msg.ID, nil
+		}
+		b.Log.Errorf("Could not edit webhook message: %s", err)
+	}
+
+	b.Log.Debugf("Processing webhook sending for message %#v", msg)
+	discordMsg, err := b.webhookSend(msg, channelID)
+	if err != nil {
+		b.Log.Errorf("Could not broadcast via webhook for message %#v: %s", msg, err)
+		return "", err
+	}
+	if discordMsg == nil {
+		return "", nil
+	}
+
+	return discordMsg.ID, nil
+}

bridge/helper/helper.go

@@ -48,6 +48,30 @@ func DownloadFileAuth(url string, auth string) (*[]byte, error) {
 	return &data, nil
 }
 
+// DownloadFileAuthRocket downloads the given URL using the specified Rocket user ID and authentication token.
+func DownloadFileAuthRocket(url, token, userID string) (*[]byte, error) {
+	var buf bytes.Buffer
+	client := &http.Client{
+		Timeout: time.Second * 5,
+	}
+	req, err := http.NewRequest("GET", url, nil)
+
+	req.Header.Add("X-Auth-Token", token)
+	req.Header.Add("X-User-Id", userID)
+
+	if err != nil {
+		return nil, err
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	_, err = io.Copy(&buf, resp.Body)
+	data := buf.Bytes()
+	return &data, err
+}
+
 // GetSubLines splits messages in newline-delimited lines. If maxLineLength is
 // specified as non-zero GetSubLines will also clip long lines to the maximum
 // length and insert a warning marker that the line was clipped.
@@ -55,8 +79,10 @@ func DownloadFileAuth(url string, auth string) (*[]byte, error) {
 // TODO: The current implementation has the inconvenient that it disregards
 // word boundaries when splitting but this is hard to solve without potentially
 // breaking formatting and other stylistic effects.
-func GetSubLines(message string, maxLineLength int) []string {
+func GetSubLines(message string, maxLineLength int, clippingMessage string) []string {
-	const clippingMessage = " <clipped message>"
+	if clippingMessage == "" {
+		clippingMessage = " <clipped message>"
+	}
 
 	var lines []string
 	for _, line := range strings.Split(strings.TrimSpace(message), "\n") {
@@ -166,8 +192,11 @@ func RemoveEmptyNewLines(msg string) string {
 
 // ClipMessage trims a message to the specified length if it exceeds it and adds a warning
 // to the message in case it does so.
-func ClipMessage(text string, length int) string {
+func ClipMessage(text string, length int, clippingMessage string) string {
-	const clippingMessage = " <clipped message>"
+	if clippingMessage == "" {
+		clippingMessage = " <clipped message>"
+	}
+
 	if len(text) > length {
 		text = text[:length-len(clippingMessage)]
 		if r, size := utf8.DecodeLastRuneInString(text); r == utf8.RuneError {
@@ -180,7 +209,7 @@ func ClipMessage(text string, length int) string {
 
 // ParseMarkdown takes in an input string as markdown and parses it to html
 func ParseMarkdown(input string) string {
-	extensions := parser.HardLineBreak | parser.NoIntraEmphasis
+	extensions := parser.HardLineBreak | parser.NoIntraEmphasis | parser.FencedCode
 	markdownParser := parser.NewWithExtensions(extensions)
 	renderer := html.NewRenderer(html.RendererOptions{
 		Flags: 0,
@@ -192,7 +221,7 @@ func ParseMarkdown(input string) string {
 	return res
 }
 
-// ConvertWebPToPNG convert input data (which should be WebP format to PNG format)
+// ConvertWebPToPNG converts input data (which should be WebP format) to PNG format
 func ConvertWebPToPNG(data *[]byte) error {
 	r := bytes.NewReader(*data)
 	m, err := webp.Decode(r)

bridge/helper/helper_test.go

@@ -10,98 +10,96 @@ import (
 
 const testLineLength = 64
 
-var (
+var lineSplittingTestCases = map[string]struct {
-	lineSplittingTestCases = map[string]struct {
+	input          string
-		input          string
+	splitOutput    []string
-		splitOutput    []string
+	nonSplitOutput []string
-		nonSplitOutput []string
+}{
-	}{
+	"Short single-line message": {
-		"Short single-line message": {
+		input:          "short",
-			input:          "short",
+		splitOutput:    []string{"short"},
-			splitOutput:    []string{"short"},
+		nonSplitOutput: []string{"short"},
-			nonSplitOutput: []string{"short"},
+	},
+	"Long single-line message": {
+		input: "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.",
+		splitOutput: []string{
+			"Lorem ipsum dolor sit amet, consectetur adipis <clipped message>",
+			"cing elit, sed do eiusmod tempor incididunt ut <clipped message>",
+			" labore et dolore magna aliqua.",
 		},
-		"Long single-line message": {
+		nonSplitOutput: []string{"Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua."},
-			input: "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.",
+	},
-			splitOutput: []string{
+	"Short multi-line message": {
-				"Lorem ipsum dolor sit amet, consectetur adipis <clipped message>",
+		input: "I\ncan't\nget\nno\nsatisfaction!",
-				"cing elit, sed do eiusmod tempor incididunt ut <clipped message>",
+		splitOutput: []string{
-				" labore et dolore magna aliqua.",
+			"I",
-			},
+			"can't",
-			nonSplitOutput: []string{"Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua."},
+			"get",
+			"no",
+			"satisfaction!",
 		},
-		"Short multi-line message": {
+		nonSplitOutput: []string{
-			input: "I\ncan't\nget\nno\nsatisfaction!",
+			"I",
-			splitOutput: []string{
+			"can't",
-				"I",
+			"get",
-				"can't",
+			"no",
-				"get",
+			"satisfaction!",
-				"no",
-				"satisfaction!",
-			},
-			nonSplitOutput: []string{
-				"I",
-				"can't",
-				"get",
-				"no",
-				"satisfaction!",
-			},
 		},
-		"Long multi-line message": {
+	},
-			input: "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.\n" +
+	"Long multi-line message": {
-				"Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.\n" +
+		input: "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.\n" +
-				"Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.\n" +
+			"Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.\n" +
-				"Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.",
+			"Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.\n" +
-			splitOutput: []string{
+			"Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.",
-				"Lorem ipsum dolor sit amet, consectetur adipis <clipped message>",
+		splitOutput: []string{
-				"cing elit, sed do eiusmod tempor incididunt ut <clipped message>",
+			"Lorem ipsum dolor sit amet, consectetur adipis <clipped message>",
-				" labore et dolore magna aliqua.",
+			"cing elit, sed do eiusmod tempor incididunt ut <clipped message>",
-				"Ut enim ad minim veniam, quis nostrud exercita <clipped message>",
+			" labore et dolore magna aliqua.",
-				"tion ullamco laboris nisi ut aliquip ex ea com <clipped message>",
+			"Ut enim ad minim veniam, quis nostrud exercita <clipped message>",
-				"modo consequat.",
+			"tion ullamco laboris nisi ut aliquip ex ea com <clipped message>",
-				"Duis aute irure dolor in reprehenderit in volu <clipped message>",
+			"modo consequat.",
-				"ptate velit esse cillum dolore eu fugiat nulla <clipped message>",
+			"Duis aute irure dolor in reprehenderit in volu <clipped message>",
-				" pariatur.",
+			"ptate velit esse cillum dolore eu fugiat nulla <clipped message>",
-				"Excepteur sint occaecat cupidatat non proident <clipped message>",
+			" pariatur.",
-				", sunt in culpa qui officia deserunt mollit an <clipped message>",
+			"Excepteur sint occaecat cupidatat non proident <clipped message>",
-				"im id est laborum.",
+			", sunt in culpa qui officia deserunt mollit an <clipped message>",
-			},
+			"im id est laborum.",
-			nonSplitOutput: []string{
-				"Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.",
-				"Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.",
-				"Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.",
-				"Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.",
-			},
 		},
-		"Message ending with new-line.": {
+		nonSplitOutput: []string{
-			input:          "Newline ending\n",
+			"Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.",
-			splitOutput:    []string{"Newline ending"},
+			"Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.",
-			nonSplitOutput: []string{"Newline ending"},
+			"Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.",
+			"Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.",
 		},
-		"Long message containing UTF-8 multi-byte runes": {
+	},
-			input: "不布人個我此而及單石業喜資富下我河下日沒一我臺空達的常景便物沒為……子大我別名解成？生賣的全直黑，我自我結毛分洲了世當，是政福那是東；斯說",
+	"Message ending with new-line.": {
-			splitOutput: []string{
+		input:          "Newline ending\n",
-				"不布人個我此而及單石業喜資富下 <clipped message>",
+		splitOutput:    []string{"Newline ending"},
-				"我河下日沒一我臺空達的常景便物 <clipped message>",
+		nonSplitOutput: []string{"Newline ending"},
-				"沒為……子大我別名解成？生賣的 <clipped message>",
+	},
-				"全直黑，我自我結毛分洲了世當， <clipped message>",
+	"Long message containing UTF-8 multi-byte runes": {
-				"是政福那是東；斯說",
+		input: "不布人個我此而及單石業喜資富下我河下日沒一我臺空達的常景便物沒為……子大我別名解成？生賣的全直黑，我自我結毛分洲了世當，是政福那是東；斯說",
-			},
+		splitOutput: []string{
-			nonSplitOutput: []string{"不布人個我此而及單石業喜資富下我河下日沒一我臺空達的常景便物沒為……子大我別名解成？生賣的全直黑，我自我結毛分洲了世當，是政福那是東；斯說"},
+			"不布人個我此而及單石業喜資富下 <clipped message>",
+			"我河下日沒一我臺空達的常景便物 <clipped message>",
+			"沒為……子大我別名解成？生賣的 <clipped message>",
+			"全直黑，我自我結毛分洲了世當， <clipped message>",
+			"是政福那是東；斯說",
 		},
-	}
+		nonSplitOutput: []string{"不布人個我此而及單石業喜資富下我河下日沒一我臺空達的常景便物沒為……子大我別名解成？生賣的全直黑，我自我結毛分洲了世當，是政福那是東；斯說"},
-)
+	},
+}
 
 func TestGetSubLines(t *testing.T) {
 	for testname, testcase := range lineSplittingTestCases {
-		splitLines := GetSubLines(testcase.input, testLineLength)
+		splitLines := GetSubLines(testcase.input, testLineLength, "")
 		assert.Equalf(t, testcase.splitOutput, splitLines, "'%s' testcase should give expected lines with splitting.", testname)
 		for _, splitLine := range splitLines {
 			byteLength := len([]byte(splitLine))
 			assert.True(t, byteLength <= testLineLength, "Splitted line '%s' of testcase '%s' should not exceed the maximum byte-length (%d vs. %d).", splitLine, testcase, byteLength, testLineLength)
 		}
 
-		nonSplitLines := GetSubLines(testcase.input, 0)
+		nonSplitLines := GetSubLines(testcase.input, 0, "")
 		assert.Equalf(t, testcase.nonSplitOutput, nonSplitLines, "'%s' testcase should give expected lines without splitting.", testname)
 	}
 }
@@ -110,16 +108,19 @@ func TestConvertWebPToPNG(t *testing.T) {
 	if os.Getenv("LOCAL_TEST") == "" {
 		t.Skip()
 	}
+
 	input, err := ioutil.ReadFile("test.webp")
 	if err != nil {
 		t.Fail()
 	}
+
 	d := &input
 	err = ConvertWebPToPNG(d)
 	if err != nil {
 		t.Fail()
 	}
-	err = ioutil.WriteFile("test.png", *d, 0644)
+
+	err = ioutil.WriteFile("test.png", *d, 0o644) // nolint:gosec
 	if err != nil {
 		t.Fail()
 	}

bridge/helper/libtgsconverter.go

@@ -0,0 +1,36 @@
+//go:build cgo
+// +build cgo
+
+package helper
+
+import (
+	"fmt"
+
+	"github.com/Benau/tgsconverter/libtgsconverter"
+	"github.com/sirupsen/logrus"
+)
+
+func CanConvertTgsToX() error {
+	return nil
+}
+
+// ConvertTgsToX convert input data (which should be tgs format) to any format supported by libtgsconverter
+func ConvertTgsToX(data *[]byte, outputFormat string, logger *logrus.Entry) error {
+	options := libtgsconverter.NewConverterOptions()
+	options.SetExtension(outputFormat)
+	blob, err := libtgsconverter.ImportFromData(*data, options)
+	if err != nil {
+		return fmt.Errorf("failed to run libtgsconverter.ImportFromData: %s", err.Error())
+	}
+
+	*data = blob
+	return nil
+}
+
+func SupportsFormat(format string) bool {
+	return libtgsconverter.SupportsExtension(format)
+}
+
+func LottieBackend() string {
+	return "libtgsconverter"
+}

bridge/helper/lottie_convert.go

@@ -0,0 +1,89 @@
+// +build !cgo
+
+package helper
+
+import (
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"github.com/sirupsen/logrus"
+)
+
+// CanConvertTgsToX Checks whether the external command necessary for ConvertTgsToX works.
+func CanConvertTgsToX() error {
+	// We depend on the fact that `lottie_convert.py --help` has exit status 0.
+	// Hyrum's Law predicted this, and Murphy's Law predicts that this will break eventually.
+	// However, there is no alternative like `lottie_convert.py --is-properly-installed`
+	cmd := exec.Command("lottie_convert.py", "--help")
+	return cmd.Run()
+}
+
+// ConvertTgsToWebP convert input data (which should be tgs format) to WebP format
+// This relies on an external command, which is ugly, but works.
+func ConvertTgsToX(data *[]byte, outputFormat string, logger *logrus.Entry) error {
+	// lottie can't handle input from a pipe, so write to a temporary file:
+	tmpInFile, err := ioutil.TempFile(os.TempDir(), "matterbridge-lottie-input-*.tgs")
+	if err != nil {
+		return err
+	}
+	tmpInFileName := tmpInFile.Name()
+	defer func() {
+		if removeErr := os.Remove(tmpInFileName); removeErr != nil {
+			logger.Errorf("Could not delete temporary (input) file %s: %v", tmpInFileName, removeErr)
+		}
+	}()
+	// lottie can handle writing to a pipe, but there is no way to do that platform-independently.
+	// "/dev/stdout" won't work on Windows, and "-" upsets Cairo for some reason. So we need another file:
+	tmpOutFile, err := ioutil.TempFile(os.TempDir(), "matterbridge-lottie-output-*.data")
+	if err != nil {
+		return err
+	}
+	tmpOutFileName := tmpOutFile.Name()
+	defer func() {
+		if removeErr := os.Remove(tmpOutFileName); removeErr != nil {
+			logger.Errorf("Could not delete temporary (output) file %s: %v", tmpOutFileName, removeErr)
+		}
+	}()
+
+	if _, writeErr := tmpInFile.Write(*data); writeErr != nil {
+		return writeErr
+	}
+	// Must close before calling lottie to avoid data races:
+	if closeErr := tmpInFile.Close(); closeErr != nil {
+		return closeErr
+	}
+
+	// Call lottie to transform:
+	cmd := exec.Command("lottie_convert.py", "--input-format", "lottie", "--output-format", outputFormat, tmpInFileName, tmpOutFileName)
+	cmd.Stdout = nil
+	cmd.Stderr = nil
+	// NB: lottie writes progress into to stderr in all cases.
+	_, stderr := cmd.Output()
+	if stderr != nil {
+		// 'stderr' already contains some parts of Stderr, because it was set to 'nil'.
+		return stderr
+	}
+	dataContents, err := ioutil.ReadFile(tmpOutFileName)
+	if err != nil {
+		return err
+	}
+
+	*data = dataContents
+	return nil
+}
+
+func SupportsFormat(format string) bool {
+	switch format {
+	case "png":
+		fallthrough
+	case "webp":
+		return true
+	default:
+		return false
+	}
+	return false
+}
+
+func LottieBackend() string {
+	return "lottie_convert.py"
+}

bridge/irc/handlers.go

@@ -67,6 +67,20 @@ func (b *Birc) handleFiles(msg *config.Message) bool {
 	return true
 }
 
+func (b *Birc) handleInvite(client *girc.Client, event girc.Event) {
+	if len(event.Params) != 2 {
+		return
+	}
+
+	channel := event.Params[1]
+
+	b.Log.Debugf("got invite for %s", channel)
+
+	if _, ok := b.channels[channel]; ok {
+		b.i.Cmd.Join(channel)
+	}
+}
+
 func (b *Birc) handleJoinPart(client *girc.Client, event girc.Event) {
 	if len(event.Params) == 0 {
 		b.Log.Debugf("handleJoinPart: empty Params? %#v", event)
@@ -109,14 +123,15 @@ func (b *Birc) handleNewConnection(client *girc.Client, event girc.Event) {
 	i := b.i
 	b.Nick = event.Params[0]
 
-	i.Handlers.Add("PRIVMSG", b.handlePrivMsg)
+	i.Handlers.AddBg("PRIVMSG", b.handlePrivMsg)
-	i.Handlers.Add("CTCP_ACTION", b.handlePrivMsg)
+	i.Handlers.AddBg("CTCP_ACTION", b.handlePrivMsg)
 	i.Handlers.Add(girc.RPL_TOPICWHOTIME, b.handleTopicWhoTime)
-	i.Handlers.Add(girc.NOTICE, b.handleNotice)
+	i.Handlers.AddBg(girc.NOTICE, b.handleNotice)
-	i.Handlers.Add("JOIN", b.handleJoinPart)
+	i.Handlers.AddBg("JOIN", b.handleJoinPart)
-	i.Handlers.Add("PART", b.handleJoinPart)
+	i.Handlers.AddBg("PART", b.handleJoinPart)
-	i.Handlers.Add("QUIT", b.handleJoinPart)
+	i.Handlers.AddBg("QUIT", b.handleJoinPart)
-	i.Handlers.Add("KICK", b.handleJoinPart)
+	i.Handlers.AddBg("KICK", b.handleJoinPart)
+	i.Handlers.Add("INVITE", b.handleInvite)
 }
 
 func (b *Birc) handleNickServ() {
@@ -170,7 +185,14 @@ func (b *Birc) handlePrivMsg(client *girc.Client, event girc.Event) {
 	if b.skipPrivMsg(event) {
 		return
 	}
-	rmsg := config.Message{Username: event.Source.Name, Channel: strings.ToLower(event.Params[0]), Account: b.Account, UserID: event.Source.Ident + "@" + event.Source.Host}
+
+	rmsg := config.Message{
+		Username: event.Source.Name,
+		Channel:  strings.ToLower(event.Params[0]),
+		Account:  b.Account,
+		UserID:   event.Source.Ident + "@" + event.Source.Host,
+	}
+
 	b.Log.Debugf("== Receiving PRIVMSG: %s %s %#v", event.Source.Name, event.Last(), event)
 
 	// set action event
@@ -178,6 +200,11 @@ func (b *Birc) handlePrivMsg(client *girc.Client, event girc.Event) {
 		rmsg.Event = config.EventUserAction
 	}
 
+	// set NOTICE event
+	if event.Command == "NOTICE" {
+		rmsg.Event = config.EventNoticeIRC
+	}
+
 	// strip action, we made an event if it was an action
 	rmsg.Text += event.StripAction()
 

bridge/irc/irc.go

@@ -4,6 +4,7 @@ import (
 	"crypto/tls"
 	"fmt"
 	"hash/crc32"
+	"io/ioutil"
 	"net"
 	"sort"
 	"strconv"
@@ -29,6 +30,7 @@ type Birc struct {
 	Local                                     chan config.Message // local queue for flood control
 	FirstConnection, authDone                 bool
 	MessageDelay, MessageQueue, MessageLength int
+	channels                                  map[string]bool
 
 	*bridge.Config
 }
@@ -39,6 +41,8 @@ func New(cfg *bridge.Config) bridge.Bridger {
 	b.Nick = b.GetString("Nick")
 	b.names = make(map[string][]string)
 	b.connected = make(chan error)
+	b.channels = make(map[string]bool)
+
 	if b.GetInt("MessageDelay") == 0 {
 		b.MessageDelay = 1300
 	} else {
@@ -111,6 +115,7 @@ func (b *Birc) Disconnect() error {
 }
 
 func (b *Birc) JoinChannel(channel config.ChannelInfo) error {
+	b.channels[channel.Name] = true
 	// need to check if we have nickserv auth done before joining channels
 	for {
 		if b.authDone {
@@ -162,9 +167,9 @@ func (b *Birc) Send(msg config.Message) (string, error) {
 	}
 
 	if b.GetBool("MessageSplit") {
-		msgLines = helper.GetSubLines(msg.Text, b.MessageLength)
+		msgLines = helper.GetSubLines(msg.Text, b.MessageLength, b.GetString("MessageClipped"))
 	} else {
-		msgLines = helper.GetSubLines(msg.Text, 0)
+		msgLines = helper.GetSubLines(msg.Text, 0, b.GetString("MessageClipped"))
 	}
 	for i := range msgLines {
 		if len(b.Local) >= b.MessageQueue {
@@ -200,22 +205,58 @@ func (b *Birc) doConnect() {
 	}
 }
 
+// Sanitize nicks for RELAYMSG: replace IRC characters with special meanings with "-"
+func sanitizeNick(nick string) string {
+	sanitize := func(r rune) rune {
+		if strings.ContainsRune("!+%@&#$:'\"?*,. ", r) {
+			return '-'
+		}
+		return r
+	}
+	return strings.Map(sanitize, nick)
+}
+
 func (b *Birc) doSend() {
 	rate := time.Millisecond * time.Duration(b.MessageDelay)
 	throttle := time.NewTicker(rate)
 	for msg := range b.Local {
 		<-throttle.C
 		username := msg.Username
-		if b.GetBool("Colornicks") && len(username) > 1 {
+		// Optional support for the proposed RELAYMSG extension, described at
-			checksum := crc32.ChecksumIEEE([]byte(msg.Username))
+		// https://github.com/jlu5/ircv3-specifications/blob/master/extensions/relaymsg.md
-			colorCode := checksum%14 + 2 // quick fix - prevent white or black color codes
+		// nolint:nestif
-			username = fmt.Sprintf("\x03%02d%s\x0F", colorCode, msg.Username)
+		if (b.i.HasCapability("overdrivenetworks.com/relaymsg") || b.i.HasCapability("draft/relaymsg")) &&
-		}
+			b.GetBool("UseRelayMsg") {
-		if msg.Event == config.EventUserAction {
+			username = sanitizeNick(username)
-			b.i.Cmd.Action(msg.Channel, username+msg.Text)
+			text := msg.Text
+
+			// Work around girc chomping leading commas on single word messages?
+			if strings.HasPrefix(text, ":") && !strings.ContainsRune(text, ' ') {
+				text = ":" + text
+			}
+
+			if msg.Event == config.EventUserAction {
+				b.i.Cmd.SendRawf("RELAYMSG %s %s :\x01ACTION %s\x01", msg.Channel, username, text) //nolint:errcheck
+			} else {
+				b.Log.Debugf("Sending RELAYMSG to channel %s: nick=%s", msg.Channel, username)
+				b.i.Cmd.SendRawf("RELAYMSG %s %s :%s", msg.Channel, username, text) //nolint:errcheck
+			}
 		} else {
-			b.Log.Debugf("Sending to channel %s", msg.Channel)
+			if b.GetBool("Colornicks") {
-			b.i.Cmd.Message(msg.Channel, username+msg.Text)
+				checksum := crc32.ChecksumIEEE([]byte(msg.Username))
+				colorCode := checksum%14 + 2 // quick fix - prevent white or black color codes
+				username = fmt.Sprintf("\x03%02d%s\x0F", colorCode, msg.Username)
+			}
+			switch msg.Event {
+			case config.EventUserAction:
+				b.i.Cmd.Action(msg.Channel, username+msg.Text)
+			case config.EventNoticeIRC:
+				b.Log.Debugf("Sending notice to channel %s", msg.Channel)
+				b.i.Cmd.Notice(msg.Channel, username+msg.Text)
+			default:
+				b.Log.Debugf("Sending to channel %s", msg.Channel)
+				b.i.Cmd.Message(msg.Channel, username+msg.Text)
+			}
 		}
 	}
 }
@@ -230,8 +271,11 @@ func (b *Birc) getClient() (*girc.Client, error) {
 	if err != nil {
 		return nil, err
 	}
+	user := b.GetString("UserName")
+	if user == "" {
+		user = b.GetString("Nick")
+	}
 	// fix strict user handling of girc
-	user := b.GetString("Nick")
 	for !girc.IsValidUser(user) {
 		if len(user) == 1 || len(user) == 0 {
 			user = "matterbridge"
@@ -239,6 +283,22 @@ func (b *Birc) getClient() (*girc.Client, error) {
 		}
 		user = user[1:]
 	}
+	realName := b.GetString("RealName")
+	if realName == "" {
+		realName = b.GetString("Nick")
+	}
+
+	debug := ioutil.Discard
+	if b.GetInt("DebugLevel") == 2 {
+		debug = b.Log.Writer()
+	}
+
+	pingDelay, err := time.ParseDuration(b.GetString("pingdelay"))
+	if err != nil || pingDelay == 0 {
+		pingDelay = time.Minute
+	}
+
+	b.Log.Debugf("setting pingdelay to %s", pingDelay)
 
 	i := girc.New(girc.Config{
 		Server:     server,
@@ -246,12 +306,14 @@ func (b *Birc) getClient() (*girc.Client, error) {
 		Port:       port,
 		Nick:       b.GetString("Nick"),
 		User:       user,
-		Name:       b.GetString("Nick"),
+		Name:       realName,
 		SSL:        b.GetBool("UseTLS"),
 		TLSConfig:  &tls.Config{InsecureSkipVerify: b.GetBool("SkipTLSVerify"), ServerName: server}, //nolint:gosec
-		PingDelay:  time.Minute,
+		PingDelay:  pingDelay,
 		// skip gIRC internal rate limiting, since we have our own throttling
-		AllowFlood: true,
+		AllowFlood:    true,
+		Debug:         debug,
+		SupportedCaps: map[string][]string{"overdrivenetworks.com/relaymsg": nil, "draft/relaymsg": nil},
 	})
 	return i, nil
 }
@@ -261,12 +323,16 @@ func (b *Birc) endNames(client *girc.Client, event girc.Event) {
 	sort.Strings(b.names[channel])
 	maxNamesPerPost := (300 / b.nicksPerRow()) * b.nicksPerRow()
 	for len(b.names[channel]) > maxNamesPerPost {
-		b.Remote <- config.Message{Username: b.Nick, Text: b.formatnicks(b.names[channel][0:maxNamesPerPost]),
+		b.Remote <- config.Message{
-			Channel: channel, Account: b.Account}
+			Username: b.Nick, Text: b.formatnicks(b.names[channel][0:maxNamesPerPost]),
+			Channel: channel, Account: b.Account,
+		}
 		b.names[channel] = b.names[channel][maxNamesPerPost:]
 	}
-	b.Remote <- config.Message{Username: b.Nick, Text: b.formatnicks(b.names[channel]),
+	b.Remote <- config.Message{
-		Channel: channel, Account: b.Account}
+		Username: b.Nick, Text: b.formatnicks(b.names[channel]),
+		Channel: channel, Account: b.Account,
+	}
 	b.names[channel] = nil
 	b.i.Handlers.Clear(girc.RPL_NAMREPLY)
 	b.i.Handlers.Clear(girc.RPL_ENDOFNAMES)
@@ -277,7 +343,7 @@ func (b *Birc) skipPrivMsg(event girc.Event) bool {
 	b.Nick = b.i.GetNick()
 
 	// freenode doesn't send 001 as first reply
-	if event.Command == "NOTICE" {
+	if event.Command == "NOTICE" && len(event.Params) != 2 {
 		return true
 	}
 	// don't forward queries to the bot
@@ -288,6 +354,15 @@ func (b *Birc) skipPrivMsg(event girc.Event) bool {
 	if event.Source.Name == b.Nick {
 		return true
 	}
+	// don't forward messages we sent via RELAYMSG
+	if relayedNick, ok := event.Tags.Get("draft/relaymsg"); ok && relayedNick == b.Nick {
+		return true
+	}
+	// This is the old name of the cap sent in spoofed messages; I've kept this in
+	// for compatibility reasons
+	if relayedNick, ok := event.Tags.Get("relaymsg"); ok && relayedNick == b.Nick {
+		return true
+	}
 	return false
 }
 

bridge/matrix/helpers.go

@@ -0,0 +1,215 @@
+package bmatrix
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"html"
+	"strings"
+	"time"
+
+	matrix "github.com/matrix-org/gomatrix"
+)
+
+func newMatrixUsername(username string) *matrixUsername {
+	mUsername := new(matrixUsername)
+
+	// check if we have a </tag>. if we have, we don't escape HTML. #696
+	if htmlTag.MatchString(username) {
+		mUsername.formatted = username
+		// remove the HTML formatting for beautiful push messages #1188
+		mUsername.plain = htmlReplacementTag.ReplaceAllString(username, "")
+	} else {
+		mUsername.formatted = html.EscapeString(username)
+		mUsername.plain = username
+	}
+
+	return mUsername
+}
+
+// getRoomID retrieves a matching room ID from the channel name.
+func (b *Bmatrix) getRoomID(channel string) string {
+	b.RLock()
+	defer b.RUnlock()
+	for ID, name := range b.RoomMap {
+		if name == channel {
+			return ID
+		}
+	}
+
+	return ""
+}
+
+// interface2Struct marshals and immediately unmarshals an interface.
+// Useful for converting map[string]interface{} to a struct.
+func interface2Struct(in interface{}, out interface{}) error {
+	jsonObj, err := json.Marshal(in)
+	if err != nil {
+		return err //nolint:wrapcheck
+	}
+
+	return json.Unmarshal(jsonObj, out)
+}
+
+// getDisplayName retrieves the displayName for mxid, querying the homserver if the mxid is not in the cache.
+func (b *Bmatrix) getDisplayName(mxid string) string {
+	if b.GetBool("UseUserName") {
+		return mxid[1:]
+	}
+
+	b.RLock()
+	if val, present := b.NicknameMap[mxid]; present {
+		b.RUnlock()
+
+		return val.displayName
+	}
+	b.RUnlock()
+
+	displayName, err := b.mc.GetDisplayName(mxid)
+	var httpError *matrix.HTTPError
+	if errors.As(err, &httpError) {
+		b.Log.Warnf("Couldn't retrieve the display name for %s", mxid)
+	}
+
+	if err != nil {
+		return b.cacheDisplayName(mxid, mxid[1:])
+	}
+
+	return b.cacheDisplayName(mxid, displayName.DisplayName)
+}
+
+// cacheDisplayName stores the mapping between a mxid and a display name, to be reused later without performing a query to the homserver.
+// Note that old entries are cleaned when this function is called.
+func (b *Bmatrix) cacheDisplayName(mxid string, displayName string) string {
+	now := time.Now()
+
+	// scan to delete old entries, to stop memory usage from becoming too high with old entries.
+	// In addition, we also detect if another user have the same username, and if so, we append their mxids to their usernames to differentiate them.
+	toDelete := []string{}
+	conflict := false
+
+	b.Lock()
+	for mxid, v := range b.NicknameMap {
+		// to prevent username reuse across matrix servers - or even on the same server, append
+		// the mxid to the username when there is a conflict
+		if v.displayName == displayName {
+			conflict = true
+			// TODO: it would be nice to be able to rename previous messages from this user.
+			// The current behavior is that only users with clashing usernames and *that have spoken since the bridge last started* will get their mxids shown, and I don't know if that's the expected behavior.
+			v.displayName = fmt.Sprintf("%s (%s)", displayName, mxid)
+			b.NicknameMap[mxid] = v
+		}
+
+		if now.Sub(v.lastUpdated) > 10*time.Minute {
+			toDelete = append(toDelete, mxid)
+		}
+	}
+
+	if conflict {
+		displayName = fmt.Sprintf("%s (%s)", displayName, mxid)
+	}
+
+	for _, v := range toDelete {
+		delete(b.NicknameMap, v)
+	}
+
+	b.NicknameMap[mxid] = NicknameCacheEntry{
+		displayName: displayName,
+		lastUpdated: now,
+	}
+	b.Unlock()
+
+	return displayName
+}
+
+// handleError converts errors into httpError.
+//nolint:exhaustivestruct
+func handleError(err error) *httpError {
+	var mErr matrix.HTTPError
+	if !errors.As(err, &mErr) {
+		return &httpError{
+			Err: "not a HTTPError",
+		}
+	}
+
+	var httpErr httpError
+
+	if err := json.Unmarshal(mErr.Contents, &httpErr); err != nil {
+		return &httpError{
+			Err: "unmarshal failed",
+		}
+	}
+
+	return &httpErr
+}
+
+func (b *Bmatrix) containsAttachment(content map[string]interface{}) bool {
+	// Skip empty messages
+	if content["msgtype"] == nil {
+		return false
+	}
+
+	// Only allow image,video or file msgtypes
+	if !(content["msgtype"].(string) == "m.image" ||
+		content["msgtype"].(string) == "m.video" ||
+		content["msgtype"].(string) == "m.file") {
+		return false
+	}
+
+	return true
+}
+
+// getAvatarURL returns the avatar URL of the specified sender.
+func (b *Bmatrix) getAvatarURL(sender string) string {
+	urlPath := b.mc.BuildURL("profile", sender, "avatar_url")
+
+	s := struct {
+		AvatarURL string `json:"avatar_url"`
+	}{}
+
+	err := b.mc.MakeRequest("GET", urlPath, nil, &s)
+	if err != nil {
+		b.Log.Errorf("getAvatarURL failed: %s", err)
+
+		return ""
+	}
+
+	url := strings.ReplaceAll(s.AvatarURL, "mxc://", b.GetString("Server")+"/_matrix/media/r0/thumbnail/")
+	if url != "" {
+		url += "?width=37&height=37&method=crop"
+	}
+
+	return url
+}
+
+// handleRatelimit handles the ratelimit errors and return if we're ratelimited and the amount of time to sleep
+func (b *Bmatrix) handleRatelimit(err error) (time.Duration, bool) {
+	httpErr := handleError(err)
+	if httpErr.Errcode != "M_LIMIT_EXCEEDED" {
+		return 0, false
+	}
+
+	b.Log.Debugf("ratelimited: %s", httpErr.Err)
+	b.Log.Infof("getting ratelimited by matrix, sleeping approx %d seconds before retrying", httpErr.RetryAfterMs/1000)
+
+	return time.Duration(httpErr.RetryAfterMs) * time.Millisecond, true
+}
+
+// retry function will check if we're ratelimited and retries again when backoff time expired
+// returns original error if not 429 ratelimit
+func (b *Bmatrix) retry(f func() error) error {
+	b.rateMutex.Lock()
+	defer b.rateMutex.Unlock()
+
+	for {
+		if err := f(); err != nil {
+			if backoff, ok := b.handleRatelimit(err); ok {
+				time.Sleep(backoff)
+			} else {
+				return err
+			}
+		} else {
+			return nil
+		}
+	}
+}

bridge/matrix/matrix.go

@@ -3,52 +3,105 @@ package bmatrix
 import (
 	"bytes"
 	"fmt"
-	"html"
 	"mime"
 	"regexp"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/42wim/matterbridge/bridge"
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/42wim/matterbridge/bridge/helper"
-	matrix "github.com/matterbridge/gomatrix"
+	matrix "github.com/matrix-org/gomatrix"
 )
 
+var (
+	htmlTag            = regexp.MustCompile("</.*?>")
+	htmlReplacementTag = regexp.MustCompile("<[^>]*>")
+)
+
+type NicknameCacheEntry struct {
+	displayName string
+	lastUpdated time.Time
+}
+
 type Bmatrix struct {
-	mc      *matrix.Client
+	mc          *matrix.Client
-	UserID  string
+	UserID      string
-	RoomMap map[string]string
+	NicknameMap map[string]NicknameCacheEntry
+	RoomMap     map[string]string
+	rateMutex   sync.RWMutex
 	sync.RWMutex
-	htmlTag *regexp.Regexp
 	*bridge.Config
 }
 
+type httpError struct {
+	Errcode      string `json:"errcode"`
+	Err          string `json:"error"`
+	RetryAfterMs int    `json:"retry_after_ms"`
+}
+
+type matrixUsername struct {
+	plain     string
+	formatted string
+}
+
+// SubTextMessage represents the new content of the message in edit messages.
+type SubTextMessage struct {
+	MsgType string `json:"msgtype"`
+	Body    string `json:"body"`
+}
+
+// MessageRelation explains how the current message relates to a previous message.
+// Notably used for message edits.
+type MessageRelation struct {
+	EventID string `json:"event_id"`
+	Type    string `json:"rel_type"`
+}
+
+type EditedMessage struct {
+	NewContent SubTextMessage  `json:"m.new_content"`
+	RelatedTo  MessageRelation `json:"m.relates_to"`
+	matrix.TextMessage
+}
+
 func New(cfg *bridge.Config) bridge.Bridger {
 	b := &Bmatrix{Config: cfg}
-	b.htmlTag = regexp.MustCompile("</.*?>")
 	b.RoomMap = make(map[string]string)
+	b.NicknameMap = make(map[string]NicknameCacheEntry)
 	return b
 }
 
 func (b *Bmatrix) Connect() error {
 	var err error
 	b.Log.Infof("Connecting %s", b.GetString("Server"))
-	b.mc, err = matrix.NewClient(b.GetString("Server"), "", "")
+	if b.GetString("MxID") != "" && b.GetString("Token") != "" {
-	if err != nil {
+		b.mc, err = matrix.NewClient(
-		return err
+			b.GetString("Server"), b.GetString("MxID"), b.GetString("Token"),
+		)
+		if err != nil {
+			return err
+		}
+		b.UserID = b.GetString("MxID")
+		b.Log.Info("Using existing Matrix credentials")
+	} else {
+		b.mc, err = matrix.NewClient(b.GetString("Server"), "", "")
+		if err != nil {
+			return err
+		}
+		resp, err := b.mc.Login(&matrix.ReqLogin{
+			Type:       "m.login.password",
+			User:       b.GetString("Login"),
+			Password:   b.GetString("Password"),
+			Identifier: matrix.NewUserIdentifier(b.GetString("Login")),
+		})
+		if err != nil {
+			return err
+		}
+		b.mc.SetCredentials(resp.UserID, resp.AccessToken)
+		b.UserID = resp.UserID
+		b.Log.Info("Connection succeeded")
 	}
-	resp, err := b.mc.Login(&matrix.ReqLogin{
-		Type:     "m.login.password",
-		User:     b.GetString("Login"),
-		Password: b.GetString("Password"),
-	})
-	if err != nil {
-		return err
-	}
-	b.mc.SetCredentials(resp.UserID, resp.AccessToken)
-	b.UserID = resp.UserID
-	b.Log.Info("Connection succeeded")
 	go b.handlematrix()
 	return nil
 }
@@ -58,14 +111,18 @@ func (b *Bmatrix) Disconnect() error {
 }
 
 func (b *Bmatrix) JoinChannel(channel config.ChannelInfo) error {
-	resp, err := b.mc.JoinRoom(channel.Name, "", nil)
+	return b.retry(func() error {
-	if err != nil {
+		resp, err := b.mc.JoinRoom(channel.Name, "", nil)
-		return err
+		if err != nil {
-	}
+			return err
-	b.Lock()
+		}
-	b.RoomMap[resp.RoomID] = channel.Name
+
-	b.Unlock()
+		b.Lock()
-	return err
+		b.RoomMap[resp.RoomID] = channel.Name
+		b.Unlock()
+
+		return nil
+	})
 }
 
 func (b *Bmatrix) Send(msg config.Message) (string, error) {
@@ -74,17 +131,30 @@ func (b *Bmatrix) Send(msg config.Message) (string, error) {
 	channel := b.getRoomID(msg.Channel)
 	b.Log.Debugf("Channel %s maps to channel id %s", msg.Channel, channel)
 
+	username := newMatrixUsername(msg.Username)
+
 	// Make a action /me of the message
 	if msg.Event == config.EventUserAction {
 		m := matrix.TextMessage{
-			MsgType: "m.emote",
+			MsgType:       "m.emote",
-			Body:    msg.Username + msg.Text,
+			Body:          username.plain + msg.Text,
+			FormattedBody: username.formatted + msg.Text,
 		}
-		resp, err := b.mc.SendMessageEvent(channel, "m.room.message", m)
+
-		if err != nil {
+		msgID := ""
-			return "", err
+
-		}
+		err := b.retry(func() error {
-		return resp.EventID, err
+			resp, err := b.mc.SendMessageEvent(channel, "m.room.message", m)
+			if err != nil {
+				return err
+			}
+
+			msgID = resp.EventID
+
+			return err
+		})
+
+		return msgID, err
 	}
 
 	// Delete message
@@ -92,17 +162,34 @@ func (b *Bmatrix) Send(msg config.Message) (string, error) {
 		if msg.ID == "" {
 			return "", nil
 		}
-		resp, err := b.mc.RedactEvent(channel, msg.ID, &matrix.ReqRedact{})
+
-		if err != nil {
+		msgID := ""
-			return "", err
+
-		}
+		err := b.retry(func() error {
-		return resp.EventID, err
+			resp, err := b.mc.RedactEvent(channel, msg.ID, &matrix.ReqRedact{})
+			if err != nil {
+				return err
+			}
+
+			msgID = resp.EventID
+
+			return err
+		})
+
+		return msgID, err
 	}
 
 	// Upload a file if it exists
 	if msg.Extra != nil {
 		for _, rmsg := range helper.HandleExtra(&msg, b.General) {
-			if _, err := b.mc.SendText(channel, rmsg.Username+rmsg.Text); err != nil {
+			rmsg := rmsg
+
+			err := b.retry(func() error {
+				_, err := b.mc.SendText(channel, rmsg.Username+rmsg.Text)
+
+				return err
+			})
+			if err != nil {
 				b.Log.Errorf("sendText failed: %s", err)
 			}
 		}
@@ -113,55 +200,110 @@ func (b *Bmatrix) Send(msg config.Message) (string, error) {
 	}
 
 	// Edit message if we have an ID
-	// matrix has no editing support
+	if msg.ID != "" {
+		rmsg := EditedMessage{TextMessage: matrix.TextMessage{
+			Body:    username.plain + msg.Text,
+			MsgType: "m.text",
+		}}
+		if b.GetBool("HTMLDisable") {
+			rmsg.TextMessage.FormattedBody = username.formatted + "* " + msg.Text
+		} else {
+			rmsg.Format = "org.matrix.custom.html"
+			rmsg.TextMessage.FormattedBody = username.formatted + "* " + helper.ParseMarkdown(msg.Text)
+		}
+		rmsg.NewContent = SubTextMessage{
+			Body:    rmsg.TextMessage.Body,
+			MsgType: "m.text",
+		}
+		rmsg.RelatedTo = MessageRelation{
+			EventID: msg.ID,
+			Type:    "m.replace",
+		}
 
-	// Use notices to send join/leave events
+		err := b.retry(func() error {
-	if msg.Event == config.EventJoinLeave {
+			_, err := b.mc.SendMessageEvent(channel, "m.room.message", rmsg)
-		resp, err := b.mc.SendNotice(channel, msg.Username+msg.Text)
+
+			return err
+		})
 		if err != nil {
 			return "", err
 		}
+
+		return msg.ID, nil
+	}
+
+	// Use notices to send join/leave events
+	if msg.Event == config.EventJoinLeave {
+		m := matrix.TextMessage{
+			MsgType:       "m.notice",
+			Body:          username.plain + msg.Text,
+			FormattedBody: username.formatted + msg.Text,
+		}
+
+		var (
+			resp *matrix.RespSendEvent
+			err  error
+		)
+
+		err = b.retry(func() error {
+			resp, err = b.mc.SendMessageEvent(channel, "m.room.message", m)
+
+			return err
+		})
+		if err != nil {
+			return "", err
+		}
+
 		return resp.EventID, err
 	}
 
 	if b.GetBool("HTMLDisable") {
-		resp, err := b.mc.SendText(channel, msg.Username+msg.Text)
+		var (
+			resp *matrix.RespSendEvent
+			err  error
+		)
+
+		err = b.retry(func() error {
+			resp, err = b.mc.SendText(channel, username.plain+msg.Text)
+
+			return err
+		})
 		if err != nil {
 			return "", err
 		}
+
 		return resp.EventID, err
 	}
 
-	username := html.EscapeString(msg.Username)
-	// check if we have a </tag>. if we have, we don't escape HTML. #696
-	if b.htmlTag.MatchString(msg.Username) {
-		username = msg.Username
-	}
 	// Post normal message with HTML support (eg riot.im)
-	resp, err := b.mc.SendHTML(channel, msg.Username+msg.Text, username+helper.ParseMarkdown(msg.Text))
+	var (
+		resp *matrix.RespSendEvent
+		err  error
+	)
+
+	err = b.retry(func() error {
+		resp, err = b.mc.SendFormattedText(channel, username.plain+msg.Text,
+			username.formatted+helper.ParseMarkdown(msg.Text))
+
+		return err
+	})
 	if err != nil {
 		return "", err
 	}
-	return resp.EventID, err
-}
 
-func (b *Bmatrix) getRoomID(channel string) string {
+	return resp.EventID, err
-	b.RLock()
-	defer b.RUnlock()
-	for ID, name := range b.RoomMap {
-		if name == channel {
-			return ID
-		}
-	}
-	return ""
 }
 
 func (b *Bmatrix) handlematrix() {
 	syncer := b.mc.Syncer.(*matrix.DefaultSyncer)
 	syncer.OnEventType("m.room.redaction", b.handleEvent)
 	syncer.OnEventType("m.room.message", b.handleEvent)
+	syncer.OnEventType("m.room.member", b.handleMemberChange)
 	go func() {
 		for {
+			if b == nil {
+				return
+			}
 			if err := b.mc.Sync(); err != nil {
 				b.Log.Println("Sync() returned ", err)
 			}
@@ -169,6 +311,45 @@ func (b *Bmatrix) handlematrix() {
 	}()
 }
 
+func (b *Bmatrix) handleEdit(ev *matrix.Event, rmsg config.Message) bool {
+	relationInterface, present := ev.Content["m.relates_to"]
+	newContentInterface, present2 := ev.Content["m.new_content"]
+	if !(present && present2) {
+		return false
+	}
+
+	var relation MessageRelation
+	if err := interface2Struct(relationInterface, &relation); err != nil {
+		b.Log.Warnf("Couldn't parse 'm.relates_to' object with value %#v", relationInterface)
+		return false
+	}
+
+	var newContent SubTextMessage
+	if err := interface2Struct(newContentInterface, &newContent); err != nil {
+		b.Log.Warnf("Couldn't parse 'm.new_content' object with value %#v", newContentInterface)
+		return false
+	}
+
+	if relation.Type != "m.replace" {
+		return false
+	}
+
+	rmsg.ID = relation.EventID
+	rmsg.Text = newContent.Body
+	b.Remote <- rmsg
+
+	return true
+}
+
+func (b *Bmatrix) handleMemberChange(ev *matrix.Event) {
+	// Update the displayname on join messages, according to https://matrix.org/docs/spec/client_server/r0.6.1#events-on-change-of-profile-information
+	if ev.Content["membership"] == "join" {
+		if dn, ok := ev.Content["displayname"].(string); ok {
+			b.cacheDisplayName(ev.Sender, dn)
+		}
+	}
+}
+
 func (b *Bmatrix) handleEvent(ev *matrix.Event) {
 	b.Log.Debugf("== Receiving event: %#v", ev)
 	if ev.Sender != b.UserID {
@@ -182,7 +363,7 @@ func (b *Bmatrix) handleEvent(ev *matrix.Event) {
 
 		// Create our message
 		rmsg := config.Message{
-			Username: ev.Sender[1:],
+			Username: b.getDisplayName(ev.Sender),
 			Channel:  channel,
 			Account:  b.Account,
 			UserID:   ev.Sender,
@@ -190,13 +371,6 @@ func (b *Bmatrix) handleEvent(ev *matrix.Event) {
 			Avatar:   b.getAvatarURL(ev.Sender),
 		}
 
-		// Text must be a string
-		if rmsg.Text, ok = ev.Content["body"].(string); !ok {
-			b.Log.Errorf("Content[body] is not a string: %T\n%#v",
-				ev.Content["body"], ev.Content)
-			return
-		}
-
 		// Remove homeserver suffix if configured
 		if b.GetBool("NoHomeServerSuffix") {
 			re := regexp.MustCompile("(.*?):.*")
@@ -212,11 +386,23 @@ func (b *Bmatrix) handleEvent(ev *matrix.Event) {
 			return
 		}
 
+		// Text must be a string
+		if rmsg.Text, ok = ev.Content["body"].(string); !ok {
+			b.Log.Errorf("Content[body] is not a string: %T\n%#v",
+				ev.Content["body"], ev.Content)
+			return
+		}
+
 		// Do we have a /me action
 		if ev.Content["msgtype"].(string) == "m.emote" {
 			rmsg.Event = config.EventUserAction
 		}
 
+		// Is it an edit?
+		if b.handleEdit(ev, rmsg) {
+			return
+		}
+
 		// Do we have attachments
 		if b.containsAttachment(ev.Content) {
 			err := b.handleDownloadFile(&rmsg, ev.Content)
@@ -227,6 +413,11 @@ func (b *Bmatrix) handleEvent(ev *matrix.Event) {
 
 		b.Log.Debugf("<= Sending message from %s on %s to gateway", ev.Sender, b.Account)
 		b.Remote <- rmsg
+
+		// not crucial, so no ratelimit check here
+		if err := b.mc.MarkRead(ev.RoomID, ev.ID); err != nil {
+			b.Log.Errorf("couldn't mark message as read %s", err.Error())
+		}
 	}
 }
 
@@ -301,27 +492,30 @@ func (b *Bmatrix) handleUploadFiles(msg *config.Message, channel string) (string
 
 // handleUploadFile handles native upload of a file.
 func (b *Bmatrix) handleUploadFile(msg *config.Message, channel string, fi *config.FileInfo) {
+	username := newMatrixUsername(msg.Username)
 	content := bytes.NewReader(*fi.Data)
 	sp := strings.Split(fi.Name, ".")
 	mtype := mime.TypeByExtension("." + sp[len(sp)-1])
-	if !(strings.Contains(mtype, "image") || strings.Contains(mtype, "video") ||
+	// image and video uploads send no username, we have to do this ourself here #715
-		strings.Contains(mtype, "application") || strings.Contains(mtype, "audio")) {
+	err := b.retry(func() error {
-		return
+		_, err := b.mc.SendFormattedText(channel, username.plain+fi.Comment, username.formatted+fi.Comment)
-	}
+
-	if fi.Comment != "" {
+		return err
-		_, err := b.mc.SendText(channel, msg.Username+fi.Comment)
+	})
-		if err != nil {
+	if err != nil {
-			b.Log.Errorf("file comment failed: %#v", err)
+		b.Log.Errorf("file comment failed: %#v", err)
-		}
-	} else {
-		// image and video uploads send no username, we have to do this ourself here #715
-		_, err := b.mc.SendText(channel, msg.Username)
-		if err != nil {
-			b.Log.Errorf("file comment failed: %#v", err)
-		}
 	}
+
 	b.Log.Debugf("uploading file: %s %s", fi.Name, mtype)
-	res, err := b.mc.UploadToContentRepo(content, mtype, int64(len(*fi.Data)))
+
+	var res *matrix.RespMediaUpload
+
+	err = b.retry(func() error {
+		res, err = b.mc.UploadToContentRepo(content, mtype, int64(len(*fi.Data)))
+
+		return err
+	})
+
 	if err != nil {
 		b.Log.Errorf("file upload failed: %#v", err)
 		return
@@ -330,58 +524,60 @@ func (b *Bmatrix) handleUploadFile(msg *config.Message, channel string, fi *conf
 	switch {
 	case strings.Contains(mtype, "video"):
 		b.Log.Debugf("sendVideo %s", res.ContentURI)
-		_, err = b.mc.SendVideo(channel, fi.Name, res.ContentURI)
+		err = b.retry(func() error {
+			_, err = b.mc.SendVideo(channel, fi.Name, res.ContentURI)
+
+			return err
+		})
 		if err != nil {
 			b.Log.Errorf("sendVideo failed: %#v", err)
 		}
 	case strings.Contains(mtype, "image"):
 		b.Log.Debugf("sendImage %s", res.ContentURI)
-		_, err = b.mc.SendImage(channel, fi.Name, res.ContentURI)
+		err = b.retry(func() error {
+			_, err = b.mc.SendImage(channel, fi.Name, res.ContentURI)
+
+			return err
+		})
 		if err != nil {
 			b.Log.Errorf("sendImage failed: %#v", err)
 		}
-	case strings.Contains(mtype, "application"):
-		b.Log.Debugf("sendFile %s", res.ContentURI)
-		_, err = b.mc.SendFile(channel, fi.Name, res.ContentURI, mtype, uint(len(*fi.Data)))
-		if err != nil {
-			b.Log.Errorf("sendFile failed: %#v", err)
-		}
 	case strings.Contains(mtype, "audio"):
 		b.Log.Debugf("sendAudio %s", res.ContentURI)
-		_, err = b.mc.SendAudio(channel, fi.Name, res.ContentURI, mtype, uint(len(*fi.Data)))
+		err = b.retry(func() error {
+			_, err = b.mc.SendMessageEvent(channel, "m.room.message", matrix.AudioMessage{
+				MsgType: "m.audio",
+				Body:    fi.Name,
+				URL:     res.ContentURI,
+				Info: matrix.AudioInfo{
+					Mimetype: mtype,
+					Size:     uint(len(*fi.Data)),
+				},
+			})
+
+			return err
+		})
 		if err != nil {
 			b.Log.Errorf("sendAudio failed: %#v", err)
 		}
+	default:
+		b.Log.Debugf("sendFile %s", res.ContentURI)
+		err = b.retry(func() error {
+			_, err = b.mc.SendMessageEvent(channel, "m.room.message", matrix.FileMessage{
+				MsgType: "m.file",
+				Body:    fi.Name,
+				URL:     res.ContentURI,
+				Info: matrix.FileInfo{
+					Mimetype: mtype,
+					Size:     uint(len(*fi.Data)),
+				},
+			})
+
+			return err
+		})
+		if err != nil {
+			b.Log.Errorf("sendFile failed: %#v", err)
+		}
 	}
 	b.Log.Debugf("result: %#v", res)
 }
-
-// skipMessages returns true if this message should not be handled
-func (b *Bmatrix) containsAttachment(content map[string]interface{}) bool {
-	// Skip empty messages
-	if content["msgtype"] == nil {
-		return false
-	}
-
-	// Only allow image,video or file msgtypes
-	if !(content["msgtype"].(string) == "m.image" ||
-		content["msgtype"].(string) == "m.video" ||
-		content["msgtype"].(string) == "m.file") {
-		return false
-	}
-	return true
-}
-
-// getAvatarURL returns the avatar URL of the specified sender
-func (b *Bmatrix) getAvatarURL(sender string) string {
-	mxcURL, err := b.mc.GetSenderAvatarURL(sender)
-	if err != nil {
-		b.Log.Errorf("getAvatarURL failed: %s", err)
-		return ""
-	}
-	url := strings.ReplaceAll(mxcURL, "mxc://", b.GetString("Server")+"/_matrix/media/r0/thumbnail/")
-	if url != "" {
-		url += "?width=37&height=37&method=crop"
-	}
-	return url
-}

bridge/matrix/matrix_test.go

@@ -0,0 +1,28 @@
+package bmatrix
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestPlainUsername(t *testing.T) {
+	uut := newMatrixUsername("MyUser")
+
+	assert.Equal(t, "MyUser", uut.formatted)
+	assert.Equal(t, "MyUser", uut.plain)
+}
+
+func TestHTMLUsername(t *testing.T) {
+	uut := newMatrixUsername("<b>MyUser</b>")
+
+	assert.Equal(t, "<b>MyUser</b>", uut.formatted)
+	assert.Equal(t, "MyUser", uut.plain)
+}
+
+func TestFancyUsername(t *testing.T) {
+	uut := newMatrixUsername("<MyUser>")
+
+	assert.Equal(t, "&lt;MyUser&gt;", uut.formatted)
+	assert.Equal(t, "<MyUser>", uut.plain)
+}

bridge/mattermost/handlers.go

@@ -4,7 +4,9 @@ import (
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/42wim/matterbridge/bridge/helper"
 	"github.com/42wim/matterbridge/matterclient"
-	"github.com/mattermost/mattermost-server/model"
+	matterclient6 "github.com/matterbridge/matterclient"
+	"github.com/mattermost/mattermost-server/v5/model"
+	model6 "github.com/mattermost/mattermost-server/v6/model"
 )
 
 // handleDownloadAvatar downloads the avatar of userid from channel
@@ -21,12 +23,26 @@ func (b *Bmattermost) handleDownloadAvatar(userid string, channel string) {
 		Extra:    make(map[string][]interface{}),
 	}
 	if _, ok := b.avatarMap[userid]; !ok {
-		data, resp := b.mc.Client.GetProfileImage(userid, "")
+		var (
-		if resp.Error != nil {
+			data []byte
-			b.Log.Errorf("ProfileImage download failed for %#v %s", userid, resp.Error)
+			err  error
-			return
+			resp *model.Response
+		)
+		if b.mc6 != nil {
+			data, _, err = b.mc6.Client.GetProfileImage(userid, "")
+			if err != nil {
+				b.Log.Errorf("ProfileImage download failed for %#v %s", userid, err)
+				return
+			}
+		} else {
+			data, resp = b.mc.Client.GetProfileImage(userid, "")
+			if resp.Error != nil {
+				b.Log.Errorf("ProfileImage download failed for %#v %s", userid, resp.Error)
+				return
+			}
 		}
-		err := helper.HandleDownloadSize(b.Log, &rmsg, userid+".png", int64(len(data)), b.General)
+
+		err = helper.HandleDownloadSize(b.Log, &rmsg, userid+".png", int64(len(data)), b.General)
 		if err != nil {
 			b.Log.Error(err)
 			return
@@ -38,6 +54,10 @@ func (b *Bmattermost) handleDownloadAvatar(userid string, channel string) {
 
 // handleDownloadFile handles file download
 func (b *Bmattermost) handleDownloadFile(rmsg *config.Message, id string) error {
+	if b.mc6 != nil {
+		return b.handleDownloadFile6(rmsg, id)
+	}
+
 	url, _ := b.mc.Client.GetFileLink(id)
 	finfo, resp := b.mc.Client.GetFileInfo(id)
 	if resp.Error != nil {
@@ -55,6 +75,25 @@ func (b *Bmattermost) handleDownloadFile(rmsg *config.Message, id string) error
 	return nil
 }
 
+// nolint:wrapcheck
+func (b *Bmattermost) handleDownloadFile6(rmsg *config.Message, id string) error {
+	url, _, _ := b.mc6.Client.GetFileLink(id)
+	finfo, _, err := b.mc6.Client.GetFileInfo(id)
+	if err != nil {
+		return err
+	}
+	err = helper.HandleDownloadSize(b.Log, rmsg, finfo.Name, finfo.Size, b.General)
+	if err != nil {
+		return err
+	}
+	data, _, err := b.mc6.Client.DownloadFile(id, true)
+	if err != nil {
+		return err
+	}
+	helper.HandleDownloadData(b.Log, rmsg, finfo.Name, rmsg.Text, url, &data, b.General)
+	return nil
+}
+
 func (b *Bmattermost) handleMatter() {
 	messages := make(chan *config.Message)
 	if b.GetString("WebhookBindAddress") != "" {
@@ -87,6 +126,12 @@ func (b *Bmattermost) handleMatter() {
 }
 
 func (b *Bmattermost) handleMatterClient(messages chan *config.Message) {
+	if b.mc6 != nil {
+		b.Log.Debug("starting matterclient6")
+		b.handleMatterClient6(messages)
+		return
+	}
+
 	for message := range b.mc.MessageChan {
 		b.Log.Debugf("%#v", message.Raw.Data)
 
@@ -108,7 +153,7 @@ func (b *Bmattermost) handleMatterClient(messages chan *config.Message) {
 			Channel:  message.Channel,
 			Text:     message.Text,
 			ID:       message.Post.Id,
-			ParentID: message.Post.ParentId,
+			ParentID: message.Post.RootId, // ParentID is obsolete with mattermost
 			Extra:    make(map[string][]interface{}),
 		}
 
@@ -140,6 +185,61 @@ func (b *Bmattermost) handleMatterClient(messages chan *config.Message) {
 	}
 }
 
+// nolint:cyclop
+func (b *Bmattermost) handleMatterClient6(messages chan *config.Message) {
+	for message := range b.mc6.MessageChan {
+		b.Log.Debugf("%#v", message.Raw.GetData())
+
+		if b.skipMessage6(message) {
+			b.Log.Debugf("Skipped message: %#v", message)
+			continue
+		}
+
+		// only download avatars if we have a place to upload them (configured mediaserver)
+		if b.General.MediaServerUpload != "" || b.General.MediaDownloadPath != "" {
+			b.handleDownloadAvatar(message.UserID, message.Channel)
+		}
+
+		b.Log.Debugf("== Receiving event %#v", message)
+
+		rmsg := &config.Message{
+			Username: message.Username,
+			UserID:   message.UserID,
+			Channel:  message.Channel,
+			Text:     message.Text,
+			ID:       message.Post.Id,
+			ParentID: message.Post.RootId, // ParentID is obsolete with mattermost
+			Extra:    make(map[string][]interface{}),
+		}
+
+		// handle mattermost post properties (override username and attachments)
+		b.handleProps6(rmsg, message)
+
+		// create a text for bridges that don't support native editing
+		if message.Raw.EventType() == model6.WebsocketEventPostEdited && !b.GetBool("EditDisable") {
+			rmsg.Text = message.Text + b.GetString("EditSuffix")
+		}
+
+		if message.Raw.EventType() == model6.WebsocketEventPostDeleted {
+			rmsg.Event = config.EventMsgDelete
+		}
+
+		for _, id := range message.Post.FileIds {
+			err := b.handleDownloadFile(rmsg, id)
+			if err != nil {
+				b.Log.Errorf("download failed: %s", err)
+			}
+		}
+
+		// Use nickname instead of username if defined
+		if nick := b.mc6.GetNickName(rmsg.UserID); nick != "" {
+			rmsg.Username = nick
+		}
+
+		messages <- rmsg
+	}
+}
+
 func (b *Bmattermost) handleMatterHook(messages chan *config.Message) {
 	for {
 		message := b.mh.Receive()
@@ -155,6 +255,10 @@ func (b *Bmattermost) handleMatterHook(messages chan *config.Message) {
 
 // handleUploadFile handles native upload of files
 func (b *Bmattermost) handleUploadFile(msg *config.Message) (string, error) {
+	if b.mc6 != nil {
+		return b.handleUploadFile6(msg)
+	}
+
 	var err error
 	var res, id string
 	channelID := b.mc.GetChannelId(msg.Channel, b.TeamID)
@@ -173,6 +277,26 @@ func (b *Bmattermost) handleUploadFile(msg *config.Message) (string, error) {
 	return res, err
 }
 
+// nolint:forcetypeassert,wrapcheck
+func (b *Bmattermost) handleUploadFile6(msg *config.Message) (string, error) {
+	var err error
+	var res, id string
+	channelID := b.mc6.GetChannelID(msg.Channel, b.TeamID)
+	for _, f := range msg.Extra["file"] {
+		fi := f.(config.FileInfo)
+		id, err = b.mc6.UploadFile(*fi.Data, channelID, fi.Name)
+		if err != nil {
+			return "", err
+		}
+		msg.Text = fi.Comment
+		if b.GetBool("PrefixMessagesWithNick") {
+			msg.Text = msg.Username + msg.Text
+		}
+		res, err = b.mc6.PostMessageWithFiles(channelID, msg.Text, msg.ParentID, []string{id})
+	}
+	return res, err
+}
+
 func (b *Bmattermost) handleProps(rmsg *config.Message, message *matterclient.Message) {
 	props := message.Post.Props
 	if props == nil {
@@ -197,3 +321,31 @@ func (b *Bmattermost) handleProps(rmsg *config.Message, message *matterclient.Me
 		}
 	}
 }
+
+// nolint:forcetypeassert
+func (b *Bmattermost) handleProps6(rmsg *config.Message, message *matterclient6.Message) {
+	props := message.Post.Props
+	if props == nil {
+		return
+	}
+	if _, ok := props["override_username"].(string); ok {
+		rmsg.Username = props["override_username"].(string)
+	}
+	if _, ok := props["attachments"].([]interface{}); ok {
+		rmsg.Extra["attachments"] = props["attachments"].([]interface{})
+		if rmsg.Text != "" {
+			return
+		}
+
+		for _, attachment := range rmsg.Extra["attachments"] {
+			attach := attachment.(map[string]interface{})
+			if attach["text"].(string) != "" {
+				rmsg.Text += attach["text"].(string)
+				continue
+			}
+			if attach["fallback"].(string) != "" {
+				rmsg.Text += attach["fallback"].(string)
+			}
+		}
+	}
+}

bridge/mattermost/helpers.go

@@ -1,13 +1,16 @@
 package bmattermost
 
 import (
+	"net/http"
 	"strings"
 
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/42wim/matterbridge/bridge/helper"
 	"github.com/42wim/matterbridge/matterclient"
 	"github.com/42wim/matterbridge/matterhook"
-	"github.com/mattermost/mattermost-server/model"
+	matterclient6 "github.com/matterbridge/matterclient"
+	"github.com/mattermost/mattermost-server/v5/model"
+	model6 "github.com/mattermost/mattermost-server/v6/model"
 )
 
 func (b *Bmattermost) doConnectWebhookBind() error {
@@ -15,25 +18,47 @@ func (b *Bmattermost) doConnectWebhookBind() error {
 	case b.GetString("WebhookURL") != "":
 		b.Log.Info("Connecting using webhookurl (sending) and webhookbindaddress (receiving)")
 		b.mh = matterhook.New(b.GetString("WebhookURL"),
-			matterhook.Config{InsecureSkipVerify: b.GetBool("SkipTLSVerify"),
+			matterhook.Config{
-				BindAddress: b.GetString("WebhookBindAddress")})
+				InsecureSkipVerify: b.GetBool("SkipTLSVerify"),
+				BindAddress:        b.GetString("WebhookBindAddress"),
+			})
 	case b.GetString("Token") != "":
 		b.Log.Info("Connecting using token (sending)")
-		err := b.apiLogin()
+		b.Log.Infof("Using mattermost v6 methods: %t", b.v6)
-		if err != nil {
+
-			return err
+		if b.v6 {
+			err := b.apiLogin6()
+			if err != nil {
+				return err
+			}
+		} else {
+			err := b.apiLogin()
+			if err != nil {
+				return err
+			}
 		}
 	case b.GetString("Login") != "":
 		b.Log.Info("Connecting using login/password (sending)")
-		err := b.apiLogin()
+		b.Log.Infof("Using mattermost v6 methods: %t", b.v6)
-		if err != nil {
+
-			return err
+		if b.v6 {
+			err := b.apiLogin6()
+			if err != nil {
+				return err
+			}
+		} else {
+			err := b.apiLogin()
+			if err != nil {
+				return err
+			}
 		}
 	default:
 		b.Log.Info("Connecting using webhookbindaddress (receiving)")
 		b.mh = matterhook.New(b.GetString("WebhookURL"),
-			matterhook.Config{InsecureSkipVerify: b.GetBool("SkipTLSVerify"),
+			matterhook.Config{
-				BindAddress: b.GetString("WebhookBindAddress")})
+				InsecureSkipVerify: b.GetBool("SkipTLSVerify"),
+				BindAddress:        b.GetString("WebhookBindAddress"),
+			})
 	}
 	return nil
 }
@@ -41,19 +66,39 @@ func (b *Bmattermost) doConnectWebhookBind() error {
 func (b *Bmattermost) doConnectWebhookURL() error {
 	b.Log.Info("Connecting using webhookurl (sending)")
 	b.mh = matterhook.New(b.GetString("WebhookURL"),
-		matterhook.Config{InsecureSkipVerify: b.GetBool("SkipTLSVerify"),
+		matterhook.Config{
-			DisableServer: true})
+			InsecureSkipVerify: b.GetBool("SkipTLSVerify"),
+			DisableServer:      true,
+		})
 	if b.GetString("Token") != "" {
 		b.Log.Info("Connecting using token (receiving)")
-		err := b.apiLogin()
+		b.Log.Infof("Using mattermost v6 methods: %t", b.v6)
-		if err != nil {
+
-			return err
+		if b.v6 {
+			err := b.apiLogin6()
+			if err != nil {
+				return err
+			}
+		} else {
+			err := b.apiLogin()
+			if err != nil {
+				return err
+			}
 		}
 	} else if b.GetString("Login") != "" {
 		b.Log.Info("Connecting using login/password (receiving)")
-		err := b.apiLogin()
+		b.Log.Infof("Using mattermost v6 methods: %t", b.v6)
-		if err != nil {
+
-			return err
+		if b.v6 {
+			err := b.apiLogin6()
+			if err != nil {
+				return err
+			}
+		} else {
+			err := b.apiLogin()
+			if err != nil {
+				return err
+			}
 		}
 	}
 	return nil
@@ -84,6 +129,31 @@ func (b *Bmattermost) apiLogin() error {
 	return nil
 }
 
+// nolint:wrapcheck
+func (b *Bmattermost) apiLogin6() error {
+	password := b.GetString("Password")
+	if b.GetString("Token") != "" {
+		password = "token=" + b.GetString("Token")
+	}
+
+	b.mc6 = matterclient6.New(b.GetString("Login"), password, b.GetString("Team"), b.GetString("Server"), "")
+	if b.GetBool("debug") {
+		b.mc6.SetLogLevel("debug")
+	}
+	b.mc6.SkipTLSVerify = b.GetBool("SkipTLSVerify")
+	b.mc6.SkipVersionCheck = b.GetBool("SkipVersionCheck")
+	b.mc6.NoTLS = b.GetBool("NoTLS")
+	b.Log.Infof("Connecting %s (team: %s) on %s", b.GetString("Login"), b.GetString("Team"), b.GetString("Server"))
+
+	if err := b.mc6.Login(); err != nil {
+		return err
+	}
+
+	b.Log.Info("Connection succeeded")
+	b.TeamID = b.mc6.GetTeamID()
+	return nil
+}
+
 // replaceAction replace the message with the correct action (/me) code
 func (b *Bmattermost) replaceAction(text string) (string, bool) {
 	if strings.HasPrefix(text, "*") && strings.HasSuffix(text, "*") {
@@ -223,3 +293,84 @@ func (b *Bmattermost) skipMessage(message *matterclient.Message) bool {
 	}
 	return false
 }
+
+// skipMessages returns true if this message should not be handled
+// nolint:gocyclo,cyclop
+func (b *Bmattermost) skipMessage6(message *matterclient6.Message) bool {
+	// Handle join/leave
+	if message.Type == "system_join_leave" ||
+		message.Type == "system_join_channel" ||
+		message.Type == "system_leave_channel" {
+		if b.GetBool("nosendjoinpart") {
+			return true
+		}
+		b.Log.Debugf("Sending JOIN_LEAVE event from %s to gateway", b.Account)
+		b.Remote <- config.Message{
+			Username: "system",
+			Text:     message.Text,
+			Channel:  message.Channel,
+			Account:  b.Account,
+			Event:    config.EventJoinLeave,
+		}
+		return true
+	}
+
+	// Handle edited messages
+	if (message.Raw.EventType() == model6.WebsocketEventPostEdited) && b.GetBool("EditDisable") {
+		return true
+	}
+
+	// Ignore non-post messages
+	if message.Post == nil {
+		b.Log.Debugf("ignoring nil message.Post: %#v", message)
+		return true
+	}
+
+	// Ignore messages sent from matterbridge
+	if message.Post.Props != nil {
+		if _, ok := message.Post.Props["matterbridge_"+b.uuid].(bool); ok {
+			b.Log.Debugf("sent by matterbridge, ignoring")
+			return true
+		}
+	}
+
+	// Ignore messages sent from a user logged in as the bot
+	if b.mc6.User.Username == message.Username {
+		return true
+	}
+
+	// if the message has reactions don't repost it (for now, until we can correlate reaction with message)
+	if message.Post.HasReactions {
+		return true
+	}
+
+	// ignore messages from other teams than ours
+	if message.Raw.GetData()["team_id"].(string) != b.TeamID {
+		return true
+	}
+
+	// only handle posted, edited or deleted events
+	if !(message.Raw.EventType() == "posted" || message.Raw.EventType() == model6.WebsocketEventPostEdited ||
+		message.Raw.EventType() == model6.WebsocketEventPostDeleted) {
+		return true
+	}
+	return false
+}
+
+func (b *Bmattermost) getVersion() string {
+	proto := "https"
+
+	if b.GetBool("notls") {
+		proto = "http"
+	}
+
+	resp, err := http.Get(proto + "://" + b.GetString("server"))
+	if err != nil {
+		b.Log.Error("failed getting version")
+		return ""
+	}
+
+	defer resp.Body.Close()
+
+	return resp.Header.Get("X-Version-Id")
+}

bridge/mattermost/mattermost.go

@@ -3,18 +3,22 @@ package bmattermost
 import (
 	"errors"
 	"fmt"
+	"strings"
 
 	"github.com/42wim/matterbridge/bridge"
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/42wim/matterbridge/bridge/helper"
 	"github.com/42wim/matterbridge/matterclient"
 	"github.com/42wim/matterbridge/matterhook"
+	matterclient6 "github.com/matterbridge/matterclient"
 	"github.com/rs/xid"
 )
 
 type Bmattermost struct {
 	mh     *matterhook.Client
 	mc     *matterclient.MMClient
+	mc6    *matterclient6.Client
+	v6     bool
 	uuid   string
 	TeamID string
 	*bridge.Config
@@ -25,7 +29,10 @@ const mattermostPlugin = "mattermost.plugin"
 
 func New(cfg *bridge.Config) bridge.Bridger {
 	b := &Bmattermost{Config: cfg, avatarMap: make(map[string]string)}
+
+	b.v6 = b.GetBool("v6")
 	b.uuid = xid.New().String()
+
 	return b
 }
 
@@ -37,6 +44,13 @@ func (b *Bmattermost) Connect() error {
 	if b.Account == mattermostPlugin {
 		return nil
 	}
+
+	if strings.HasPrefix(b.getVersion(), "6.") {
+		if !b.v6 {
+			b.v6 = true
+		}
+	}
+
 	if b.GetString("WebhookBindAddress") != "" {
 		if err := b.doConnectWebhookBind(); err != nil {
 			return err
@@ -53,16 +67,34 @@ func (b *Bmattermost) Connect() error {
 		return nil
 	case b.GetString("Token") != "":
 		b.Log.Info("Connecting using token (sending and receiving)")
-		err := b.apiLogin()
+		b.Log.Infof("Using mattermost v6 methods: %t", b.v6)
-		if err != nil {
+
-			return err
+		if b.v6 {
+			err := b.apiLogin6()
+			if err != nil {
+				return err
+			}
+		} else {
+			err := b.apiLogin()
+			if err != nil {
+				return err
+			}
 		}
 		go b.handleMatter()
 	case b.GetString("Login") != "":
 		b.Log.Info("Connecting using login/password (sending and receiving)")
-		err := b.apiLogin()
+		b.Log.Infof("Using mattermost v6 methods: %t", b.v6)
-		if err != nil {
+
-			return err
+		if b.v6 {
+			err := b.apiLogin6()
+			if err != nil {
+				return err
+			}
+		} else {
+			err := b.apiLogin()
+			if err != nil {
+				return err
+			}
 		}
 		go b.handleMatter()
 	}
@@ -83,10 +115,20 @@ func (b *Bmattermost) JoinChannel(channel config.ChannelInfo) error {
 	}
 	// we can only join channels using the API
 	if b.GetString("WebhookURL") == "" && b.GetString("WebhookBindAddress") == "" {
-		id := b.mc.GetChannelId(channel.Name, b.TeamID)
+		var id string
+		if b.mc6 != nil {
+			id = b.mc6.GetChannelID(channel.Name, b.TeamID)
+		} else {
+			id = b.mc.GetChannelId(channel.Name, b.TeamID)
+		}
 		if id == "" {
 			return fmt.Errorf("Could not find channel ID for channel %s", channel.Name)
 		}
+
+		if b.mc6 != nil {
+			return b.mc6.JoinChannel(id) // nolint:wrapcheck
+		}
+
 		return b.mc.JoinChannel(id)
 	}
 	return nil
@@ -118,20 +160,47 @@ func (b *Bmattermost) Send(msg config.Message) (string, error) {
 		if msg.ID == "" {
 			return "", nil
 		}
+		if b.mc6 != nil {
+			return msg.ID, b.mc6.DeleteMessage(msg.ID) // nolint:wrapcheck
+		}
+
 		return msg.ID, b.mc.DeleteMessage(msg.ID)
 	}
 
 	// Handle prefix hint for unthreaded messages.
-	if msg.ParentID == "msg-parent-not-found" {
+	if msg.ParentNotFound() {
 		msg.ParentID = ""
 		msg.Text = fmt.Sprintf("[thread]: %s", msg.Text)
 	}
 
+	// we only can reply to the root of the thread, not to a specific ID (like discord for example does)
+	if msg.ParentID != "" {
+		if b.mc6 != nil {
+			post, _, err := b.mc6.Client.GetPost(msg.ParentID, "")
+			if err != nil {
+				b.Log.Errorf("getting post %s failed: %s", msg.ParentID, err)
+			}
+			msg.ParentID = post.RootId
+		} else {
+			post, res := b.mc.Client.GetPost(msg.ParentID, "")
+			if res.Error != nil {
+				b.Log.Errorf("getting post %s failed: %s", msg.ParentID, res.Error.DetailedError)
+			}
+			msg.ParentID = post.RootId
+		}
+	}
+
 	// Upload a file if it exists
 	if msg.Extra != nil {
 		for _, rmsg := range helper.HandleExtra(&msg, b.General) {
-			if _, err := b.mc.PostMessage(b.mc.GetChannelId(rmsg.Channel, b.TeamID), rmsg.Username+rmsg.Text, msg.ParentID); err != nil {
+			if b.mc6 != nil {
-				b.Log.Errorf("PostMessage failed: %s", err)
+				if _, err := b.mc6.PostMessage(b.mc.GetChannelId(rmsg.Channel, b.TeamID), rmsg.Username+rmsg.Text, msg.ParentID); err != nil {
+					b.Log.Errorf("PostMessage failed: %s", err)
+				}
+			} else {
+				if _, err := b.mc.PostMessage(b.mc.GetChannelId(rmsg.Channel, b.TeamID), rmsg.Username+rmsg.Text, msg.ParentID); err != nil {
+					b.Log.Errorf("PostMessage failed: %s", err)
+				}
 			}
 		}
 		if len(msg.Extra["file"]) > 0 {
@@ -146,9 +215,17 @@ func (b *Bmattermost) Send(msg config.Message) (string, error) {
 
 	// Edit message if we have an ID
 	if msg.ID != "" {
+		if b.mc6 != nil {
+			return b.mc6.EditMessage(msg.ID, msg.Text) // nolint:wrapcheck
+		}
+
 		return b.mc.EditMessage(msg.ID, msg.Text)
 	}
 
 	// Post normal message
+	if b.mc6 != nil {
+		return b.mc6.PostMessage(b.mc6.GetChannelID(msg.Channel, b.TeamID), msg.Text, msg.ParentID) // nolint:wrapcheck
+	}
+
 	return b.mc.PostMessage(b.mc.GetChannelId(msg.Channel, b.TeamID), msg.Text, msg.ParentID)
 }

bridge/msteams/msteams.go

@@ -19,8 +19,10 @@ import (
 	"golang.org/x/oauth2"
 )
 
-var defaultScopes = []string{"openid", "profile", "offline_access", "Group.Read.All", "Group.ReadWrite.All"}
+var (
-var attachRE = regexp.MustCompile(`<attachment id=.*?attachment>`)
+	defaultScopes = []string{"openid", "profile", "offline_access", "Group.Read.All", "Group.ReadWrite.All"}
+	attachRE      = regexp.MustCompile(`<attachment id=.*?attachment>`)
+)
 
 type Bmsteams struct {
 	gc    *msgraph.GraphServiceRequestBuilder
@@ -50,7 +52,7 @@ func (b *Bmsteams) Connect() error {
 		b.Log.Errorf("Couldn't save sessionfile in %s: %s", tokenCachePath, err)
 	}
 	// make file readable only for matterbridge user
-	err = os.Chmod(tokenCachePath, 0600)
+	err = os.Chmod(tokenCachePath, 0o600)
 	if err != nil {
 		b.Log.Errorf("Couldn't change permissions for %s: %s", tokenCachePath, err)
 	}
@@ -86,13 +88,16 @@ func (b *Bmsteams) JoinChannel(channel config.ChannelInfo) error {
 
 func (b *Bmsteams) Send(msg config.Message) (string, error) {
 	b.Log.Debugf("=> Receiving %#v", msg)
-	if msg.ParentID != "" && msg.ParentID != "msg-parent-not-found" {
+	if msg.ParentValid() {
 		return b.sendReply(msg)
 	}
-	if msg.ParentID == "msg-parent-not-found" {
+
+	// Handle prefix hint for unthreaded messages.
+	if msg.ParentNotFound() {
 		msg.ParentID = ""
 		msg.Text = fmt.Sprintf("[thread]: %s", msg.Text)
 	}
+
 	ct := b.gc.Teams().ID(b.GetString("TeamID")).Channels().ID(msg.Channel).Messages().Request()
 	text := msg.Username + msg.Text
 	content := &msgraph.ItemBody{Content: &text}
@@ -165,7 +170,7 @@ func (b *Bmsteams) poll(channelName string) error {
 			}
 
 			// skip non-user message for now.
-			if msg.From.User == nil {
+			if msg.From == nil || msg.From.User == nil {
 				continue
 			}
 

bridge/mumble/handlers.go

@@ -0,0 +1,96 @@
+package bmumble
+
+import (
+	"strconv"
+	"time"
+
+	"layeh.com/gumble/gumble"
+
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/42wim/matterbridge/bridge/helper"
+)
+
+func (b *Bmumble) handleServerConfig(event *gumble.ServerConfigEvent) {
+	b.serverConfigUpdate <- *event
+}
+
+func (b *Bmumble) handleTextMessage(event *gumble.TextMessageEvent) {
+	sender := "unknown"
+	if event.TextMessage.Sender != nil {
+		sender = event.TextMessage.Sender.Name
+	}
+	// If the text message is received before receiving a ServerSync
+	// and UserState, Client.Self or Self.Channel are nil
+	if event.Client.Self == nil || event.Client.Self.Channel == nil {
+		b.Log.Warn("Connection bootstrap not finished, discarding text message")
+		return
+	}
+	// Convert Mumble HTML messages to markdown
+	parts, err := b.convertHTMLtoMarkdown(event.TextMessage.Message)
+	if err != nil {
+		b.Log.Error(err)
+	}
+	now := time.Now().UTC()
+	for i, part := range parts {
+		// Construct matterbridge message and pass on to the gateway
+		rmsg := config.Message{
+			Channel:  strconv.FormatUint(uint64(event.Client.Self.Channel.ID), 10),
+			Username: sender,
+			UserID:   sender + "@" + b.Host,
+			Account:  b.Account,
+		}
+		if part.Image == nil {
+			rmsg.Text = part.Text
+		} else {
+			fname := b.Account + "_" + strconv.FormatInt(now.UnixNano(), 10) + "_" + strconv.Itoa(i) + part.FileExtension
+			rmsg.Extra = make(map[string][]interface{})
+			if err = helper.HandleDownloadSize(b.Log, &rmsg, fname, int64(len(part.Image)), b.General); err != nil {
+				b.Log.WithError(err).Warn("not including image in message")
+				continue
+			}
+			helper.HandleDownloadData(b.Log, &rmsg, fname, "", "", &part.Image, b.General)
+		}
+		b.Log.Debugf("Sending message to gateway: %+v", rmsg)
+		b.Remote <- rmsg
+	}
+}
+
+func (b *Bmumble) handleConnect(event *gumble.ConnectEvent) {
+	// Set the user's "bio"/comment
+	if comment := b.GetString("UserComment"); comment != "" && event.Client.Self != nil {
+		event.Client.Self.SetComment(comment)
+	}
+	// No need to talk or listen
+	event.Client.Self.SetSelfDeafened(true)
+	event.Client.Self.SetSelfMuted(true)
+	// if the Channel variable is set, this is a reconnect -> rejoin channel
+	if b.Channel != nil {
+		if err := b.doJoin(event.Client, *b.Channel); err != nil {
+			b.Log.Error(err)
+		}
+		b.Remote <- config.Message{
+			Username: "system",
+			Text:     "rejoin",
+			Channel:  "",
+			Account:  b.Account,
+			Event:    config.EventRejoinChannels,
+		}
+	}
+}
+
+func (b *Bmumble) handleUserChange(event *gumble.UserChangeEvent) {
+	// Only care about changes to self
+	if event.User != event.Client.Self {
+		return
+	}
+	// Someone attempted to move the user out of the configured channel; attempt to join back
+	if b.Channel != nil {
+		if err := b.doJoin(event.Client, *b.Channel); err != nil {
+			b.Log.Error(err)
+		}
+	}
+}
+
+func (b *Bmumble) handleDisconnect(event *gumble.DisconnectEvent) {
+	b.connected <- *event
+}

bridge/mumble/helpers.go

@@ -0,0 +1,143 @@
+package bmumble
+
+import (
+	"fmt"
+	"mime"
+	"net/http"
+	"regexp"
+	"strings"
+
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/mattn/godown"
+	"github.com/vincent-petithory/dataurl"
+)
+
+type MessagePart struct {
+	Text          string
+	FileExtension string
+	Image         []byte
+}
+
+func (b *Bmumble) decodeImage(uri string, parts *[]MessagePart) error {
+	// Decode the data:image/... URI
+	image, err := dataurl.DecodeString(uri)
+	if err != nil {
+		b.Log.WithError(err).Info("No image extracted")
+		return err
+	}
+	// Determine the file extensions for that image
+	ext, err := mime.ExtensionsByType(image.MediaType.ContentType())
+	if err != nil || len(ext) == 0 {
+		b.Log.WithError(err).Infof("No file extension registered for MIME type '%s'", image.MediaType.ContentType())
+		return err
+	}
+	// Add the image to the MessagePart slice
+	*parts = append(*parts, MessagePart{"", ext[0], image.Data})
+	return nil
+}
+
+func (b *Bmumble) tokenize(t *string) ([]MessagePart, error) {
+	// `^(.*?)` matches everything before the image
+	// `!\[[^\]]*\]\(` matches the `![alt](` part of markdown images
+	// `(data:image\/[^)]+)` matches the data: URI used by Mumble
+	// `\)` matches the closing parenthesis after the URI
+	// `(.*)$` matches the remaining text to be examined in the next iteration
+	p := regexp.MustCompile(`^(?ms)(.*?)!\[[^\]]*\]\((data:image\/[^)]+)\)(.*)$`)
+	remaining := *t
+	var parts []MessagePart
+	for {
+		tokens := p.FindStringSubmatch(remaining)
+		if tokens == nil {
+			// no match -> remaining string is non-image text
+			pre := strings.TrimSpace(remaining)
+			if len(pre) > 0 {
+				parts = append(parts, MessagePart{pre, "", nil})
+			}
+			return parts, nil
+		}
+
+		// tokens[1] is the text before the image
+		if len(tokens[1]) > 0 {
+			pre := strings.TrimSpace(tokens[1])
+			parts = append(parts, MessagePart{pre, "", nil})
+		}
+		// tokens[2] is the image URL
+		uri, err := dataurl.UnescapeToString(strings.TrimSpace(strings.ReplaceAll(tokens[2], " ", "")))
+		if err != nil {
+			b.Log.WithError(err).Info("URL unescaping failed")
+			remaining = strings.TrimSpace(tokens[3])
+			continue
+		}
+		err = b.decodeImage(uri, &parts)
+		if err != nil {
+			b.Log.WithError(err).Info("Decoding the image failed")
+		}
+		// tokens[3] is the text after the image, processed in the next iteration
+		remaining = strings.TrimSpace(tokens[3])
+	}
+}
+
+func (b *Bmumble) convertHTMLtoMarkdown(html string) ([]MessagePart, error) {
+	var sb strings.Builder
+	err := godown.Convert(&sb, strings.NewReader(html), nil)
+	if err != nil {
+		return nil, err
+	}
+	markdown := sb.String()
+	b.Log.Debugf("### to markdown: %s", markdown)
+	return b.tokenize(&markdown)
+}
+
+func (b *Bmumble) extractFiles(msg *config.Message) []config.Message {
+	var messages []config.Message
+	if msg.Extra == nil || len(msg.Extra["file"]) == 0 {
+		return messages
+	}
+	// Create a separate message for each file
+	for _, f := range msg.Extra["file"] {
+		fi := f.(config.FileInfo)
+		imsg := config.Message{
+			Channel:   msg.Channel,
+			Username:  msg.Username,
+			UserID:    msg.UserID,
+			Account:   msg.Account,
+			Protocol:  msg.Protocol,
+			Timestamp: msg.Timestamp,
+			Event:     "mumble_image",
+		}
+		// If no data is present for the file, send a link instead
+		if fi.Data == nil || len(*fi.Data) == 0 {
+			if len(fi.URL) > 0 {
+				imsg.Text = fmt.Sprintf(`<a href="%s">%s</a>`, fi.URL, fi.URL)
+				messages = append(messages, imsg)
+			} else {
+				b.Log.Infof("Not forwarding file without local data")
+			}
+			continue
+		}
+		mimeType := http.DetectContentType(*fi.Data)
+		// Mumble only supports images natively, send a link instead
+		if !strings.HasPrefix(mimeType, "image/") {
+			if len(fi.URL) > 0 {
+				imsg.Text = fmt.Sprintf(`<a href="%s">%s</a>`, fi.URL, fi.URL)
+				messages = append(messages, imsg)
+			} else {
+				b.Log.Infof("Not forwarding file of type %s", mimeType)
+			}
+			continue
+		}
+		mimeType = strings.TrimSpace(strings.Split(mimeType, ";")[0])
+		// Build data:image/...;base64,... style image URL and embed image directly into the message
+		du := dataurl.New(*fi.Data, mimeType)
+		dataURL, err := du.MarshalText()
+		if err != nil {
+			b.Log.WithError(err).Infof("Image Serialization into data URL failed (type: %s, length: %d)", mimeType, len(*fi.Data))
+			continue
+		}
+		imsg.Text = fmt.Sprintf(`<img src="%s"/>`, dataURL)
+		messages = append(messages, imsg)
+	}
+	// Remove files from original message
+	msg.Extra["file"] = nil
+	return messages
+}

bridge/mumble/mumble.go

@@ -0,0 +1,262 @@
+package bmumble
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"strconv"
+	"strings"
+	"time"
+
+	"layeh.com/gumble/gumble"
+	"layeh.com/gumble/gumbleutil"
+
+	"github.com/42wim/matterbridge/bridge"
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/42wim/matterbridge/bridge/helper"
+	stripmd "github.com/writeas/go-strip-markdown"
+
+	// We need to import the 'data' package as an implicit dependency.
+	// See: https://godoc.org/github.com/paulrosania/go-charset/charset
+	_ "github.com/paulrosania/go-charset/data"
+)
+
+type Bmumble struct {
+	client             *gumble.Client
+	Nick               string
+	Host               string
+	Channel            *uint32
+	local              chan config.Message
+	running            chan error
+	connected          chan gumble.DisconnectEvent
+	serverConfigUpdate chan gumble.ServerConfigEvent
+	serverConfig       gumble.ServerConfigEvent
+	tlsConfig          tls.Config
+
+	*bridge.Config
+}
+
+func New(cfg *bridge.Config) bridge.Bridger {
+	b := &Bmumble{}
+	b.Config = cfg
+	b.Nick = b.GetString("Nick")
+	b.local = make(chan config.Message)
+	b.running = make(chan error)
+	b.connected = make(chan gumble.DisconnectEvent)
+	b.serverConfigUpdate = make(chan gumble.ServerConfigEvent)
+	return b
+}
+
+func (b *Bmumble) Connect() error {
+	b.Log.Infof("Connecting %s", b.GetString("Server"))
+	host, portstr, err := net.SplitHostPort(b.GetString("Server"))
+	if err != nil {
+		return err
+	}
+	b.Host = host
+	_, err = strconv.Atoi(portstr)
+	if err != nil {
+		return err
+	}
+
+	if err = b.buildTLSConfig(); err != nil {
+		return err
+	}
+
+	go b.doSend()
+	go b.connectLoop()
+	err = <-b.running
+	return err
+}
+
+func (b *Bmumble) Disconnect() error {
+	return b.client.Disconnect()
+}
+
+func (b *Bmumble) JoinChannel(channel config.ChannelInfo) error {
+	cid, err := strconv.ParseUint(channel.Name, 10, 32)
+	if err != nil {
+		return err
+	}
+	channelID := uint32(cid)
+	if b.Channel != nil && *b.Channel != channelID {
+		b.Log.Fatalf("Cannot join channel ID '%d', already joined to channel ID %d", channelID, *b.Channel)
+		return errors.New("the Mumble bridge can only join a single channel")
+	}
+	b.Channel = &channelID
+	return b.doJoin(b.client, channelID)
+}
+
+func (b *Bmumble) Send(msg config.Message) (string, error) {
+	// Only process text messages
+	b.Log.Debugf("=> Received local message %#v", msg)
+	if msg.Event != "" && msg.Event != config.EventUserAction {
+		return "", nil
+	}
+
+	attachments := b.extractFiles(&msg)
+	b.local <- msg
+	for _, a := range attachments {
+		b.local <- a
+	}
+	return "", nil
+}
+
+func (b *Bmumble) buildTLSConfig() error {
+	b.tlsConfig = tls.Config{}
+	// Load TLS client certificate keypair required for registered user authentication
+	if cpath := b.GetString("TLSClientCertificate"); cpath != "" {
+		if ckey := b.GetString("TLSClientKey"); ckey != "" {
+			cert, err := tls.LoadX509KeyPair(cpath, ckey)
+			if err != nil {
+				return err
+			}
+			b.tlsConfig.Certificates = []tls.Certificate{cert}
+		}
+	}
+	// Load TLS CA used for server verification.  If not provided, the Go system trust anchor is used
+	if capath := b.GetString("TLSCACertificate"); capath != "" {
+		ca, err := ioutil.ReadFile(capath)
+		if err != nil {
+			return err
+		}
+		b.tlsConfig.RootCAs = x509.NewCertPool()
+		b.tlsConfig.RootCAs.AppendCertsFromPEM(ca)
+	}
+	b.tlsConfig.InsecureSkipVerify = b.GetBool("SkipTLSVerify")
+	return nil
+}
+
+func (b *Bmumble) connectLoop() {
+	firstConnect := true
+	for {
+		err := b.doConnect()
+		if firstConnect {
+			b.running <- err
+		}
+		if err != nil {
+			b.Log.Errorf("Connection to server failed: %#v", err)
+			if firstConnect {
+				break
+			} else {
+				b.Log.Info("Retrying in 10s")
+				time.Sleep(10 * time.Second)
+				continue
+			}
+		}
+		firstConnect = false
+		d := <-b.connected
+		switch d.Type {
+		case gumble.DisconnectError:
+			b.Log.Errorf("Lost connection to the server (%s), attempting reconnect", d.String)
+			continue
+		case gumble.DisconnectKicked:
+			b.Log.Errorf("Kicked from the server (%s), attempting reconnect", d.String)
+			continue
+		case gumble.DisconnectBanned:
+			b.Log.Errorf("Banned from the server (%s), not attempting reconnect", d.String)
+			close(b.connected)
+			close(b.running)
+			return
+		case gumble.DisconnectUser:
+			b.Log.Infof("Disconnect successful")
+			close(b.connected)
+			close(b.running)
+			return
+		}
+	}
+}
+
+func (b *Bmumble) doConnect() error {
+	// Create new gumble config and attach event handlers
+	gumbleConfig := gumble.NewConfig()
+	gumbleConfig.Attach(gumbleutil.Listener{
+		ServerConfig: b.handleServerConfig,
+		TextMessage:  b.handleTextMessage,
+		Connect:      b.handleConnect,
+		Disconnect:   b.handleDisconnect,
+		UserChange:   b.handleUserChange,
+	})
+	gumbleConfig.Username = b.GetString("Nick")
+	if password := b.GetString("Password"); password != "" {
+		gumbleConfig.Password = password
+	}
+
+	client, err := gumble.DialWithDialer(new(net.Dialer), b.GetString("Server"), gumbleConfig, &b.tlsConfig)
+	if err != nil {
+		return err
+	}
+	b.client = client
+	return nil
+}
+
+func (b *Bmumble) doJoin(client *gumble.Client, channelID uint32) error {
+	channel, ok := client.Channels[channelID]
+	if !ok {
+		return fmt.Errorf("no channel with ID %d", channelID)
+	}
+	client.Self.Move(channel)
+	return nil
+}
+
+func (b *Bmumble) doSend() {
+	// Message sending loop that makes sure server-side
+	// restrictions and client-side message traits don't conflict
+	// with each other.
+	for {
+		select {
+		case serverConfig := <-b.serverConfigUpdate:
+			b.Log.Debugf("Received server config update: AllowHTML=%#v, MaximumMessageLength=%#v", serverConfig.AllowHTML, serverConfig.MaximumMessageLength)
+			b.serverConfig = serverConfig
+		case msg := <-b.local:
+			b.processMessage(&msg)
+		}
+	}
+}
+
+func (b *Bmumble) processMessage(msg *config.Message) {
+	b.Log.Debugf("Processing message %s", msg.Text)
+
+	allowHTML := true
+	if b.serverConfig.AllowHTML != nil {
+		allowHTML = *b.serverConfig.AllowHTML
+	}
+
+	// If this is a specially generated image message, send it unmodified
+	if msg.Event == "mumble_image" {
+		if allowHTML {
+			b.client.Self.Channel.Send(msg.Username+msg.Text, false)
+		} else {
+			b.Log.Info("Can't send image, server does not allow HTML messages")
+		}
+		return
+	}
+
+	// Don't process empty messages
+	if len(msg.Text) == 0 {
+		return
+	}
+	// If HTML is allowed, convert markdown into HTML, otherwise strip markdown
+	if allowHTML {
+		msg.Text = helper.ParseMarkdown(msg.Text)
+	} else {
+		msg.Text = stripmd.Strip(msg.Text)
+	}
+
+	// If there is a maximum message length, split and truncate the lines
+	var msgLines []string
+	if maxLength := b.serverConfig.MaximumMessageLength; maxLength != nil {
+		msgLines = helper.GetSubLines(msg.Text, *maxLength-len(msg.Username), b.GetString("MessageClipped"))
+	} else {
+		msgLines = helper.GetSubLines(msg.Text, 0, b.GetString("MessageClipped"))
+	}
+	// Send the individual lines
+	for i := range msgLines {
+		// Remove unnecessary newline character, since either way we're sending it as individual lines
+		msgLines[i] = strings.TrimSuffix(msgLines[i], "\n")
+		b.client.Self.Channel.Send(msg.Username+msgLines[i], false)
+	}
+}

bridge/nctalk/nctalk.go

@@ -2,12 +2,13 @@ package nctalk
 
 import (
 	"context"
+	"crypto/tls"
 	"strconv"
+	"strings"
 
 	"github.com/42wim/matterbridge/bridge"
 	"github.com/42wim/matterbridge/bridge/config"
 
-	talk "gomod.garykim.dev/nc-talk"
 	"gomod.garykim.dev/nc-talk/ocs"
 	"gomod.garykim.dev/nc-talk/room"
 	"gomod.garykim.dev/nc-talk/user"
@@ -31,8 +32,18 @@ type Broom struct {
 
 func (b *Btalk) Connect() error {
 	b.Log.Info("Connecting")
-	b.user = talk.NewUser(b.GetString("Server"), b.GetString("Login"), b.GetString("Password"))
+	tconfig := &user.TalkUserConfig{
-	_, err := b.user.Capabilities()
+		TLSConfig: &tls.Config{
+			InsecureSkipVerify: b.GetBool("SkipTLSVerify"), //nolint:gosec
+		},
+	}
+	var err error
+	b.user, err = user.NewUser(b.GetString("Server"), b.GetString("Login"), b.GetString("Password"), tconfig)
+	if err != nil {
+		b.Log.Error("Config could not be used")
+		return err
+	}
+	_, err = b.user.Capabilities()
 	if err != nil {
 		b.Log.Error("Cannot Connect")
 		return err
@@ -49,8 +60,12 @@ func (b *Btalk) Disconnect() error {
 }
 
 func (b *Btalk) JoinChannel(channel config.ChannelInfo) error {
+	tr, err := room.NewTalkRoom(b.user, channel.Name)
+	if err != nil {
+		return err
+	}
 	newRoom := Broom{
-		room: talk.NewRoom(b.user, channel.Name),
+		room: tr,
 	}
 	newRoom.ctx, newRoom.ctxCancel = context.WithCancel(context.Background())
 	c, err := newRoom.room.ReceiveMessages(newRoom.ctx)
@@ -58,28 +73,34 @@ func (b *Btalk) JoinChannel(channel config.ChannelInfo) error {
 		return err
 	}
 	b.rooms = append(b.rooms, newRoom)
+
 	go func() {
 		for msg := range c {
-			// ignore messages that are one of the following
+			msg := msg
-			// * not a message from a user
+
-			// * from ourselves
+			if msg.Error != nil {
-			if msg.MessageType != ocs.MessageComment || msg.ActorID == b.user.User {
+				b.Log.Errorf("Fatal message poll error: %s\n", msg.Error)
+
+				return
+			}
+
+			// Ignore messages that are from the bot user
+			if msg.ActorID == b.user.User || msg.ActorType == "bridged" {
 				continue
 			}
-			remoteMessage := config.Message{
+
-				Text:     msg.Message,
+			// Handle deleting messages
-				Channel:  newRoom.room.Token,
+			if msg.MessageType == ocs.MessageSystem && msg.Parent != nil && msg.Parent.MessageType == ocs.MessageDelete {
-				Username: msg.ActorDisplayName,
+				b.handleDeletingMessage(&msg, &newRoom)
-				UserID:   msg.ActorID,
+				continue
-				Account:  b.Account,
 			}
-			// It is possible for the ID to not be set on older versions of Talk so we only set it if
+
-			// the ID is not blank
+			// Handle sending messages
-			if msg.ID != 0 {
+			if msg.MessageType == ocs.MessageComment {
-				remoteMessage.ID = strconv.Itoa(msg.ID)
+				b.handleSendingMessage(&msg, &newRoom)
+				continue
 			}
-			b.Log.Debugf("<= Message is %#v", remoteMessage)
+
-			b.Remote <- remoteMessage
 		}
 	}()
 	return nil
@@ -92,16 +113,40 @@ func (b *Btalk) Send(msg config.Message) (string, error) {
 		return "", nil
 	}
 
-	// Talk currently only supports sending normal messages
+	// Standard Message Send
-	if msg.Event != "" {
+	if msg.Event == "" {
-		return "", nil
+		// Handle sending files if they are included
+		err := b.handleSendingFile(&msg, r)
+		if err != nil {
+			b.Log.Errorf("Could not send files in message to room %v from %v: %v", msg.Channel, msg.Username, err)
+
+			return "", nil
+		}
+
+		sentMessage, err := b.sendText(r, &msg, msg.Text)
+		if err != nil {
+			b.Log.Errorf("Could not send message to room %v from %v: %v", msg.Channel, msg.Username, err)
+
+			return "", nil
+		}
+		return strconv.Itoa(sentMessage.ID), nil
 	}
-	sentMessage, err := r.room.SendMessage(msg.Username + msg.Text)
+
-	if err != nil {
+	// Message Deletion
-		b.Log.Errorf("Could not send message to room %v from %v: %v", msg.Channel, msg.Username, err)
+	if msg.Event == config.EventMsgDelete {
-		return "", nil
+		messageID, err := strconv.Atoi(msg.ID)
+		if err != nil {
+			return "", err
+		}
+		data, err := r.room.DeleteMessage(messageID)
+		if err != nil {
+			return "", err
+		}
+		return strconv.Itoa(data.ID), nil
 	}
-	return strconv.Itoa(sentMessage.ID), nil
+
+	// Message is not a type that is currently supported
+	return "", nil
 }
 
 func (b *Btalk) getRoom(token string) *Broom {
@@ -112,3 +157,139 @@ func (b *Btalk) getRoom(token string) *Broom {
 	}
 	return nil
 }
+
+func (b *Btalk) sendText(r *Broom, msg *config.Message, text string) (*ocs.TalkRoomMessageData, error) {
+	messageToSend := &room.Message{Message: msg.Username + text}
+
+	if b.GetBool("SeparateDisplayName") {
+		messageToSend.Message = text
+		messageToSend.ActorDisplayName = msg.Username
+	}
+
+	return r.room.SendComplexMessage(messageToSend)
+}
+
+func (b *Btalk) handleFiles(mmsg *config.Message, message *ocs.TalkRoomMessageData) error {
+	for _, parameter := range message.MessageParameters {
+		if parameter.Type == ocs.ROSTypeFile {
+			// Get the file
+			file, err := b.user.DownloadFile(parameter.Path)
+			if err != nil {
+				return err
+			}
+
+			if mmsg.Extra == nil {
+				mmsg.Extra = make(map[string][]interface{})
+			}
+
+			mmsg.Extra["file"] = append(mmsg.Extra["file"], config.FileInfo{
+				Name:   parameter.Name,
+				Data:   file,
+				Size:   int64(len(*file)),
+				Avatar: false,
+			})
+		}
+	}
+
+	return nil
+}
+
+func (b *Btalk) handleSendingFile(msg *config.Message, r *Broom) error {
+	for _, f := range msg.Extra["file"] {
+		fi := f.(config.FileInfo)
+		if fi.URL == "" {
+			continue
+		}
+
+		message := ""
+		if fi.Comment != "" {
+			message += fi.Comment + " "
+		}
+		message += fi.URL
+		_, err := b.sendText(r, msg, message)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (b *Btalk) handleSendingMessage(msg *ocs.TalkRoomMessageData, r *Broom) {
+	remoteMessage := config.Message{
+		Text:     formatRichObjectString(msg.Message, msg.MessageParameters),
+		Channel:  r.room.Token,
+		Username: DisplayName(msg, b.guestSuffix()),
+		UserID:   msg.ActorID,
+		Account:  b.Account,
+	}
+	// It is possible for the ID to not be set on older versions of Talk so we only set it if
+	// the ID is not blank
+	if msg.ID != 0 {
+		remoteMessage.ID = strconv.Itoa(msg.ID)
+	}
+
+	// Handle Files
+	err := b.handleFiles(&remoteMessage, msg)
+	if err != nil {
+		b.Log.Errorf("Error handling file: %#v", msg)
+
+		return
+	}
+
+	b.Log.Debugf("<= Message is %#v", remoteMessage)
+	b.Remote <- remoteMessage
+}
+
+func (b *Btalk) handleDeletingMessage(msg *ocs.TalkRoomMessageData, r *Broom) {
+	remoteMessage := config.Message{
+		Event:   config.EventMsgDelete,
+		Text:    config.EventMsgDelete,
+		Channel: r.room.Token,
+		ID:      strconv.Itoa(msg.Parent.ID),
+		Account: b.Account,
+	}
+	b.Log.Debugf("<= Message being deleted is %#v", remoteMessage)
+	b.Remote <- remoteMessage
+}
+
+func (b *Btalk) guestSuffix() string {
+	guestSuffix := " (Guest)"
+	if b.IsKeySet("GuestSuffix") {
+		guestSuffix = b.GetString("GuestSuffix")
+	}
+
+	return guestSuffix
+}
+
+// Spec: https://github.com/nextcloud/server/issues/1706#issue-182308785
+func formatRichObjectString(message string, parameters map[string]ocs.RichObjectString) string {
+	for id, parameter := range parameters {
+		text := parameter.Name
+
+		switch parameter.Type {
+		case ocs.ROSTypeUser, ocs.ROSTypeGroup:
+			text = "@" + text
+		case ocs.ROSTypeFile:
+			if parameter.Link != "" {
+				text = parameter.Name
+			}
+		}
+
+		message = strings.ReplaceAll(message, "{"+id+"}", text)
+	}
+
+	return message
+}
+
+func DisplayName(msg *ocs.TalkRoomMessageData, suffix string) string {
+	if msg.ActorType == ocs.ActorGuest {
+		if msg.ActorDisplayName == "" {
+			return "Guest"
+		}
+
+		return msg.ActorDisplayName + suffix
+	}
+
+	return msg.ActorDisplayName
+}

bridge/rocketchat/handlers.go

@@ -1,7 +1,10 @@
 package brocketchat
 
 import (
+	"fmt"
+
 	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/42wim/matterbridge/bridge/helper"
 	"github.com/matterbridge/Rocket.Chat.Go.SDK/models"
 )
 
@@ -58,6 +61,7 @@ func (b *Brocketchat) handleStatusEvent(ev models.Message, rmsg *config.Message)
 
 func (b *Brocketchat) handleRocketClient(messages chan *config.Message) {
 	for message := range b.messageChan {
+		message := message
 		// skip messages with same ID, apparently messages get duplicated for an unknown reason
 		if _, ok := b.cache.Get(message.ID); ok {
 			continue
@@ -76,8 +80,11 @@ func (b *Brocketchat) handleRocketClient(messages chan *config.Message) {
 			Account:  b.Account,
 			UserID:   message.User.ID,
 			ID:       message.ID,
+			Extra:    make(map[string][]interface{}),
 		}
 
+		b.handleAttachments(&message, rmsg)
+
 		// handleStatusEvent returns false if the message should be dropped
 		// in that case it is probably some modification to the channel we do not want to relay
 		if b.handleStatusEvent(m, rmsg) {
@@ -86,6 +93,38 @@ func (b *Brocketchat) handleRocketClient(messages chan *config.Message) {
 	}
 }
 
+func (b *Brocketchat) handleAttachments(message *models.Message, rmsg *config.Message) {
+	if rmsg.Text == "" {
+		for _, attachment := range message.Attachments {
+			if attachment.Title != "" {
+				rmsg.Text = attachment.Title + "\n"
+			}
+			if attachment.Title != "" && attachment.Text != "" {
+				rmsg.Text += "\n"
+			}
+			if attachment.Text != "" {
+				rmsg.Text += attachment.Text
+			}
+		}
+	}
+
+	for i := range message.Attachments {
+		if err := b.handleDownloadFile(rmsg, &message.Attachments[i]); err != nil {
+			b.Log.Errorf("Could not download incoming file: %#v", err)
+		}
+	}
+}
+
+func (b *Brocketchat) handleDownloadFile(rmsg *config.Message, file *models.Attachment) error {
+	downloadURL := b.GetString("server") + file.TitleLink
+	data, err := helper.DownloadFileAuthRocket(downloadURL, b.user.Token, b.user.ID)
+	if err != nil {
+		return fmt.Errorf("download %s failed %#v", downloadURL, err)
+	}
+	helper.HandleDownloadData(b.Log, rmsg, file.Title, rmsg.Text, downloadURL, data, b.General)
+	return nil
+}
+
 func (b *Brocketchat) handleUploadFile(msg *config.Message) error {
 	for _, f := range msg.Extra["file"] {
 		fi := f.(config.FileInfo)

bridge/slack/handlers.go

@@ -95,6 +95,8 @@ func (b *Bslack) handleSlackClient(messages chan *config.Message) {
 			b.users.populateUser(ev.User)
 		case *slack.HelloEvent, *slack.LatencyReport, *slack.ConnectingEvent:
 			continue
+		case *slack.UserChangeEvent:
+			b.users.invalidateUser(ev.User.ID)
 		default:
 			b.Log.Debugf("Unhandled incoming event: %T", ev)
 		}

bridge/slack/slack.go

@@ -156,7 +156,7 @@ func (b *Bslack) JoinChannel(channel config.ChannelInfo) error {
 
 	// try to join a channel when in legacy
 	if b.legacy {
-		_, err := b.sc.JoinChannel(channel.Name)
+		_, _, _, err := b.sc.JoinConversation(channel.Name)
 		if err != nil {
 			switch err.Error() {
 			case "name_taken", "restricted_action":
@@ -195,7 +195,7 @@ func (b *Bslack) Send(msg config.Message) (string, error) {
 		b.Log.Debugf("=> Receiving %#v", msg)
 	}
 
-	msg.Text = helper.ClipMessage(msg.Text, messageLength)
+	msg.Text = helper.ClipMessage(msg.Text, messageLength, b.GetString("MessageClipped"))
 	msg.Text = b.replaceCodeFence(msg.Text)
 
 	// Make a action /me of the message
@@ -299,7 +299,7 @@ func (b *Bslack) sendRTM(msg config.Message) (string, error) {
 	}
 
 	// Handle prefix hint for unthreaded messages.
-	if msg.ParentID == "msg-parent-not-found" {
+	if msg.ParentNotFound() {
 		msg.ParentID = ""
 		msg.Text = fmt.Sprintf("[thread]: %s", msg.Text)
 	}
@@ -459,7 +459,7 @@ func (b *Bslack) uploadFile(msg *config.Message, channelID string) {
 		b.cache.Add("filename"+fi.Name, ts)
 		initialComment := fmt.Sprintf("File from %s", msg.Username)
 		if fi.Comment != "" {
-			initialComment += fmt.Sprintf("with comment: %s", fi.Comment)
+			initialComment += fmt.Sprintf(" with comment: %s", fi.Comment)
 		}
 		res, err := b.sc.UploadFile(slack.FileUploadParameters{
 			Reader:          bytes.NewReader(*fi.Data),

bridge/slack/users_channels.go

@@ -113,6 +113,12 @@ func (b *users) populateUser(userID string) {
 	b.users[userID] = user
 }
 
+func (b *users) invalidateUser(userID string) {
+	b.usersMutex.Lock()
+	defer b.usersMutex.Unlock()
+	delete(b.users, userID)
+}
+
 func (b *users) populateUsers(wait bool) {
 	b.refreshMutex.Lock()
 	if !wait && (time.Now().Before(b.earliestRefresh) || b.refreshInProgress) {
@@ -283,7 +289,7 @@ func (b *channels) populateChannels(wait bool) {
 	// We only retrieve public and private channels, not IMs
 	// and MPIMs as those do not have a channel name.
 	queryParams := &slack.GetConversationsParameters{
-		ExcludeArchived: "true",
+		ExcludeArchived: true,
 		Types:           []string{"public_channel,private_channel"},
 	}
 	for {

bridge/telegram/handlers.go

@@ -2,7 +2,7 @@ package btelegram
 
 import (
 	"html"
-	"regexp"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"unicode/utf16"
@@ -181,13 +181,15 @@ func (b *Btelegram) handleRecv(updates <-chan tgbotapi.Update) {
 // sends a EVENT_AVATAR_DOWNLOAD message to the gateway if successful.
 // logs an error message if it fails
 func (b *Btelegram) handleDownloadAvatar(userid int, channel string) {
-	rmsg := config.Message{Username: "system",
+	rmsg := config.Message{
-		Text:    "avatar",
+		Username: "system",
-		Channel: channel,
+		Text:     "avatar",
-		Account: b.Account,
+		Channel:  channel,
-		UserID:  strconv.Itoa(userid),
+		Account:  b.Account,
-		Event:   config.EventAvatarDownload,
+		UserID:   strconv.Itoa(userid),
-		Extra:   make(map[string][]interface{})}
+		Event:    config.EventAvatarDownload,
+		Extra:    make(map[string][]interface{}),
+	}
 
 	if _, ok := b.avatarMap[strconv.Itoa(userid)]; !ok {
 		photos, err := b.c.GetUserProfilePhotos(tgbotapi.UserProfilePhotosConfig{UserID: userid, Limit: 1})
@@ -217,6 +219,36 @@ func (b *Btelegram) handleDownloadAvatar(userid int, channel string) {
 	}
 }
 
+func (b *Btelegram) maybeConvertTgs(name *string, data *[]byte) {
+	format := b.GetString("MediaConvertTgs")
+	if helper.SupportsFormat(format) {
+		b.Log.Debugf("Format supported by %s, converting %v", helper.LottieBackend(), name)
+	} else {
+		// Otherwise, no conversion was requested. Trying to run the usual webp
+		// converter would fail, because '.tgs.webp' is actually a gzipped JSON
+		// file, and has nothing to do with WebP.
+		return
+	}
+	err := helper.ConvertTgsToX(data, format, b.Log)
+	if err != nil {
+		b.Log.Errorf("conversion failed: %v", err)
+	} else {
+		*name = strings.Replace(*name, "tgs.webp", format, 1)
+	}
+}
+
+func (b *Btelegram) maybeConvertWebp(name *string, data *[]byte) {
+	if b.GetBool("MediaConvertWebPToPNG") {
+		b.Log.Debugf("WebP to PNG conversion enabled, converting %v", name)
+		err := helper.ConvertWebPToPNG(data)
+		if err != nil {
+			b.Log.Errorf("conversion failed: %v", err)
+		} else {
+			*name = strings.Replace(*name, ".webp", ".png", 1)
+		}
+	}
+}
+
 // handleDownloadFile handles file download
 func (b *Btelegram) handleDownload(rmsg *config.Message, message *tgbotapi.Message) error {
 	size := 0
@@ -264,15 +296,18 @@ func (b *Btelegram) handleDownload(rmsg *config.Message, message *tgbotapi.Messa
 	if err != nil {
 		return err
 	}
-	if strings.HasSuffix(name, ".webp") && b.GetBool("MediaConvertWebPToPNG") {
+
-		b.Log.Debugf("WebP to PNG conversion enabled, converting %s", name)
+	if strings.HasSuffix(name, ".tgs.webp") {
-		err := helper.ConvertWebPToPNG(data)
+		b.maybeConvertTgs(&name, data)
-		if err != nil {
+	} else if strings.HasSuffix(name, ".webp") {
-			b.Log.Errorf("conversion failed: %s", err)
+		b.maybeConvertWebp(&name, data)
-		} else {
-			name = strings.Replace(name, ".webp", ".png", 1)
-		}
 	}
+
+	// rename .oga to .ogg  https://github.com/42wim/matterbridge/issues/906#issuecomment-741793512
+	if strings.HasSuffix(name, ".oga") && message.Audio != nil {
+		name = strings.Replace(name, ".oga", ".ogg", 1)
+	}
+
 	helper.HandleDownloadData(b.Log, rmsg, name, message.Caption, "", data, b.General)
 	return nil
 }
@@ -346,21 +381,32 @@ func (b *Btelegram) handleUploadFile(msg *config.Message, chatid int64) string {
 			Name:  fi.Name,
 			Bytes: *fi.Data,
 		}
-		re := regexp.MustCompile(".(jpg|png)$")
+		switch filepath.Ext(fi.Name) {
-		if re.MatchString(fi.Name) {
+		case ".jpg", ".jpe", ".png":
-			c = tgbotapi.NewPhotoUpload(chatid, file)
+			pc := tgbotapi.NewPhotoUpload(chatid, file)
-		} else {
+			pc.Caption, pc.ParseMode = TGGetParseMode(b, msg.Username, fi.Comment)
-			c = tgbotapi.NewDocumentUpload(chatid, file)
+			c = pc
+		case ".mp4", ".m4v":
+			vc := tgbotapi.NewVideoUpload(chatid, file)
+			vc.Caption, vc.ParseMode = TGGetParseMode(b, msg.Username, fi.Comment)
+			c = vc
+		case ".mp3", ".oga":
+			ac := tgbotapi.NewAudioUpload(chatid, file)
+			ac.Caption, ac.ParseMode = TGGetParseMode(b, msg.Username, fi.Comment)
+			c = ac
+		case ".ogg":
+			voc := tgbotapi.NewVoiceUpload(chatid, file)
+			voc.Caption, voc.ParseMode = TGGetParseMode(b, msg.Username, fi.Comment)
+			c = voc
+		default:
+			dc := tgbotapi.NewDocumentUpload(chatid, file)
+			dc.Caption, dc.ParseMode = TGGetParseMode(b, msg.Username, fi.Comment)
+			c = dc
 		}
 		_, err := b.c.Send(c)
 		if err != nil {
 			b.Log.Errorf("file upload failed: %#v", err)
 		}
-		if fi.Comment != "" {
-			if _, err := b.sendMessage(chatid, msg.Username, fi.Comment); err != nil {
-				b.Log.Errorf("posting file comment %s failed: %s", fi.Comment, err)
-			}
-		}
 	}
 	return ""
 }
@@ -370,7 +416,7 @@ func (b *Btelegram) handleQuote(message, quoteNick, quoteMessage string) string
 	if format == "" {
 		format = "{MESSAGE} (re @{QUOTENICK}: {QUOTEMESSAGE})"
 	}
-	quoteMessagelength := len(quoteMessage)
+	quoteMessagelength := len([]rune(quoteMessage))
 	if b.GetInt("QuoteLengthLimit") != 0 && quoteMessagelength >= b.GetInt("QuoteLengthLimit") {
 		runes := []rune(quoteMessage)
 		quoteMessage = string(runes[0:b.GetInt("QuoteLengthLimit")])

bridge/telegram/telegram.go

@@ -2,6 +2,7 @@ package btelegram
 
 import (
 	"html"
+	"log"
 	"strconv"
 	"strings"
 
@@ -25,6 +26,16 @@ type Btelegram struct {
 }
 
 func New(cfg *bridge.Config) bridge.Bridger {
+	tgsConvertFormat := cfg.GetString("MediaConvertTgs")
+	if tgsConvertFormat != "" {
+		err := helper.CanConvertTgsToX()
+		if err != nil {
+			log.Fatalf("Telegram bridge configured to convert .tgs files to '%s', but %s does not appear to work:\n%#v", tgsConvertFormat, helper.LottieBackend(), err)
+		}
+		if !helper.SupportsFormat(tgsConvertFormat) {
+			log.Fatalf("Telegram bridge configured to convert .tgs files to '%s', but %s doesn't support it.", tgsConvertFormat, helper.LottieBackend())
+		}
+	}
 	return &Btelegram{Config: cfg, avatarMap: make(map[string]string)}
 }
 
@@ -56,6 +67,28 @@ func (b *Btelegram) JoinChannel(channel config.ChannelInfo) error {
 	return nil
 }
 
+func TGGetParseMode(b *Btelegram, username string, text string) (textout string, parsemode string) {
+	textout = username + text
+	if b.GetString("MessageFormat") == HTMLFormat {
+		b.Log.Debug("Using mode HTML")
+		parsemode = tgbotapi.ModeHTML
+	}
+	if b.GetString("MessageFormat") == "Markdown" {
+		b.Log.Debug("Using mode markdown")
+		parsemode = tgbotapi.ModeMarkdown
+	}
+	if b.GetString("MessageFormat") == MarkdownV2 {
+		b.Log.Debug("Using mode MarkdownV2")
+		parsemode = MarkdownV2
+	}
+	if strings.ToLower(b.GetString("MessageFormat")) == HTMLNick {
+		b.Log.Debug("Using mode HTML - nick only")
+		textout = username + html.EscapeString(text)
+		parsemode = tgbotapi.ModeHTML
+	}
+	return textout, parsemode
+}
+
 func (b *Btelegram) Send(msg config.Message) (string, error) {
 	b.Log.Debugf("=> Receiving %#v", msg)
 
@@ -118,24 +151,7 @@ func (b *Btelegram) getFileDirectURL(id string) string {
 
 func (b *Btelegram) sendMessage(chatid int64, username, text string) (string, error) {
 	m := tgbotapi.NewMessage(chatid, "")
-	m.Text = username + text
+	m.Text, m.ParseMode = TGGetParseMode(b, username, text)
-	if b.GetString("MessageFormat") == HTMLFormat {
-		b.Log.Debug("Using mode HTML")
-		m.ParseMode = tgbotapi.ModeHTML
-	}
-	if b.GetString("MessageFormat") == "Markdown" {
-		b.Log.Debug("Using mode markdown")
-		m.ParseMode = tgbotapi.ModeMarkdown
-	}
-	if b.GetString("MessageFormat") == MarkdownV2 {
-		b.Log.Debug("Using mode MarkdownV2")
-		m.ParseMode = MarkdownV2
-	}
-	if strings.ToLower(b.GetString("MessageFormat")) == HTMLNick {
-		b.Log.Debug("Using mode HTML - nick only")
-		m.Text = username + html.EscapeString(text)
-		m.ParseMode = tgbotapi.ModeHTML
-	}
 
 	m.DisableWebPagePreview = b.GetBool("DisableWebPagePreview")
 

bridge/vk/vk.go

@@ -0,0 +1,327 @@
+package bvk
+
+import (
+	"bytes"
+	"context"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/42wim/matterbridge/bridge"
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/42wim/matterbridge/bridge/helper"
+
+	"github.com/SevereCloud/vksdk/v2/api"
+	"github.com/SevereCloud/vksdk/v2/events"
+	longpoll "github.com/SevereCloud/vksdk/v2/longpoll-bot"
+	"github.com/SevereCloud/vksdk/v2/object"
+)
+
+const (
+	audioMessage = "audio_message"
+	document     = "doc"
+	photo        = "photo"
+	video        = "video"
+	graffiti     = "graffiti"
+	sticker      = "sticker"
+	wall         = "wall"
+)
+
+type user struct {
+	lastname, firstname, avatar string
+}
+
+type Bvk struct {
+	c            *api.VK
+	usernamesMap map[int]user // cache of user names and avatar URLs
+	*bridge.Config
+}
+
+func New(cfg *bridge.Config) bridge.Bridger {
+	return &Bvk{usernamesMap: make(map[int]user), Config: cfg}
+}
+
+func (b *Bvk) Connect() error {
+	b.Log.Info("Connecting")
+	b.c = api.NewVK(b.GetString("Token"))
+	lp, err := longpoll.NewLongPoll(b.c, b.GetInt("GroupID"))
+	if err != nil {
+		b.Log.Debugf("%#v", err)
+
+		return err
+	}
+
+	lp.MessageNew(func(ctx context.Context, obj events.MessageNewObject) {
+		b.handleMessage(obj.Message, false)
+	})
+
+	b.Log.Info("Connection succeeded")
+
+	go func() {
+		err := lp.Run()
+		if err != nil {
+			b.Log.Fatal("Enable longpoll in group management")
+		}
+	}()
+
+	return nil
+}
+
+func (b *Bvk) Disconnect() error {
+	return nil
+}
+
+func (b *Bvk) JoinChannel(channel config.ChannelInfo) error {
+	return nil
+}
+
+func (b *Bvk) Send(msg config.Message) (string, error) {
+	b.Log.Debugf("=> Receiving %#v", msg)
+
+	peerID, err := strconv.Atoi(msg.Channel)
+	if err != nil {
+		return "", err
+	}
+
+	params := api.Params{}
+
+	text := msg.Username + msg.Text
+
+	if msg.Extra != nil {
+		if len(msg.Extra["file"]) > 0 {
+			// generate attachments string
+			attachment, urls := b.uploadFiles(msg.Extra, peerID)
+			params["attachment"] = attachment
+			text += urls
+		}
+	}
+
+	params["message"] = text
+
+	if msg.ID == "" {
+		// New message
+		params["random_id"] = time.Now().Unix()
+		params["peer_ids"] = msg.Channel
+
+		res, e := b.c.MessagesSendPeerIDs(params)
+		if e != nil {
+			return "", err
+		}
+
+		return strconv.Itoa(res[0].ConversationMessageID), nil
+	}
+	// Edit message
+	messageID, err := strconv.ParseInt(msg.ID, 10, 64)
+	if err != nil {
+		return "", err
+	}
+
+	params["peer_id"] = peerID
+	params["conversation_message_id"] = messageID
+
+	_, err = b.c.MessagesEdit(params)
+	if err != nil {
+		return "", err
+	}
+
+	return msg.ID, nil
+}
+
+func (b *Bvk) getUser(id int) user {
+	u, found := b.usernamesMap[id]
+	if !found {
+		b.Log.Debug("Fetching username for ", id)
+
+		if id >= 0 {
+			result, _ := b.c.UsersGet(api.Params{
+				"user_ids": id,
+				"fields":   "photo_200",
+			})
+
+			resUser := result[0]
+			u = user{lastname: resUser.LastName, firstname: resUser.FirstName, avatar: resUser.Photo200}
+			b.usernamesMap[id] = u
+		} else {
+			result, _ := b.c.GroupsGetByID(api.Params{
+				"group_id": id * -1,
+			})
+
+			resGroup := result[0]
+			u = user{lastname: resGroup.Name, avatar: resGroup.Photo200}
+		}
+	}
+
+	return u
+}
+
+func (b *Bvk) handleMessage(msg object.MessagesMessage, isFwd bool) {
+	b.Log.Debug("ChatID: ", msg.PeerID)
+	// fetch user info
+	u := b.getUser(msg.FromID)
+
+	rmsg := config.Message{
+		Text:     msg.Text,
+		Username: u.firstname + " " + u.lastname,
+		Avatar:   u.avatar,
+		Channel:  strconv.Itoa(msg.PeerID),
+		Account:  b.Account,
+		UserID:   strconv.Itoa(msg.FromID),
+		ID:       strconv.Itoa(msg.ConversationMessageID),
+		Extra:    make(map[string][]interface{}),
+	}
+
+	if msg.ReplyMessage != nil {
+		ur := b.getUser(msg.ReplyMessage.FromID)
+		rmsg.Text = "Re: " + ur.firstname + " " + ur.lastname + "\n" + rmsg.Text
+	}
+
+	if isFwd {
+		rmsg.Username = "Fwd: " + rmsg.Username
+	}
+
+	if len(msg.Attachments) > 0 {
+		urls, text := b.getFiles(msg.Attachments)
+
+		if text != "" {
+			rmsg.Text += "\n" + text
+		}
+
+		// download
+		b.downloadFiles(&rmsg, urls)
+	}
+
+	if len(msg.FwdMessages) > 0 {
+		rmsg.Text += strconv.Itoa(len(msg.FwdMessages)) + " forwarded messages"
+	}
+
+	b.Remote <- rmsg
+
+	if len(msg.FwdMessages) > 0 {
+		// recursive processing of forwarded messages
+		for _, m := range msg.FwdMessages {
+			m.PeerID = msg.PeerID
+			b.handleMessage(m, true)
+		}
+	}
+}
+
+func (b *Bvk) uploadFiles(extra map[string][]interface{}, peerID int) (string, string) {
+	var attachments []string
+	text := ""
+
+	for _, f := range extra["file"] {
+		fi := f.(config.FileInfo)
+
+		if fi.Comment != "" {
+			text += fi.Comment + "\n"
+		}
+		a, err := b.uploadFile(fi, peerID)
+		if err != nil {
+			b.Log.Error("File upload error ", fi.Name)
+		}
+
+		attachments = append(attachments, a)
+	}
+
+	return strings.Join(attachments, ","), text
+}
+
+func (b *Bvk) uploadFile(file config.FileInfo, peerID int) (string, error) {
+	r := bytes.NewReader(*file.Data)
+
+	photoRE := regexp.MustCompile(".(jpg|jpe|png)$")
+	if photoRE.MatchString(file.Name) {
+		p, err := b.c.UploadMessagesPhoto(peerID, r)
+		if err != nil {
+			return "", err
+		}
+
+		return photo + strconv.Itoa(p[0].OwnerID) + "_" + strconv.Itoa(p[0].ID), nil
+	}
+
+	var doctype string
+	if strings.Contains(file.Name, ".ogg") {
+		doctype = audioMessage
+	} else {
+		doctype = document
+	}
+
+	doc, err := b.c.UploadMessagesDoc(peerID, doctype, file.Name, "", r)
+	if err != nil {
+		return "", err
+	}
+
+	switch doc.Type {
+	case audioMessage:
+		return document + strconv.Itoa(doc.AudioMessage.OwnerID) + "_" + strconv.Itoa(doc.AudioMessage.ID), nil
+	case document:
+		return document + strconv.Itoa(doc.Doc.OwnerID) + "_" + strconv.Itoa(doc.Doc.ID), nil
+	}
+
+	return "", nil
+}
+
+func (b *Bvk) getFiles(attachments []object.MessagesMessageAttachment) ([]string, string) {
+	var urls []string
+	var text []string
+
+	for _, a := range attachments {
+		switch a.Type {
+		case photo:
+			var resolution float64 = 0
+			url := a.Photo.Sizes[0].URL
+			for _, size := range a.Photo.Sizes {
+				r := size.Height * size.Width
+				if resolution < r {
+					resolution = r
+					url = size.URL
+				}
+			}
+
+			urls = append(urls, url)
+
+		case document:
+			urls = append(urls, a.Doc.URL)
+
+		case graffiti:
+			urls = append(urls, a.Graffiti.URL)
+
+		case audioMessage:
+			urls = append(urls, a.AudioMessage.DocsDocPreviewAudioMessage.LinkOgg)
+
+		case sticker:
+			var resolution float64 = 0
+			url := a.Sticker.Images[0].URL
+			for _, size := range a.Sticker.Images {
+				r := size.Height * size.Width
+				if resolution < r {
+					resolution = r
+					url = size.URL
+				}
+			}
+			urls = append(urls, url+".png")
+		case video:
+			text = append(text, "https://vk.com/video"+strconv.Itoa(a.Video.OwnerID)+"_"+strconv.Itoa(a.Video.ID))
+
+		case wall:
+			text = append(text, "https://vk.com/wall"+strconv.Itoa(a.Wall.FromID)+"_"+strconv.Itoa(a.Wall.ID))
+
+		default:
+			text = append(text, "This attachment is not supported ("+a.Type+")")
+		}
+	}
+
+	return urls, strings.Join(text, "\n")
+}
+
+func (b *Bvk) downloadFiles(rmsg *config.Message, urls []string) {
+	for _, url := range urls {
+		data, err := helper.DownloadFile(url)
+		if err == nil {
+			urlPart := strings.Split(url, "/")
+			name := strings.Split(urlPart[len(urlPart)-1], "?")[0]
+			helper.HandleDownloadData(b.Log, rmsg, name, "", url, data, b.General)
+		}
+	}
+}

bridge/whatsapp/handlers.go

@@ -24,7 +24,8 @@ Check:
 func (b *Bwhatsapp) HandleError(err error) {
 	// ignore received invalid data errors. https://github.com/42wim/matterbridge/issues/843
 	// ignore tag 174 errors. https://github.com/42wim/matterbridge/issues/1094
-	if strings.Contains(err.Error(), "error processing data: received invalid data") || strings.Contains(err.Error(), "invalid string with tag 174") {
+	if strings.Contains(err.Error(), "error processing data: received invalid data") ||
+		strings.Contains(err.Error(), "invalid string with tag 174") {
 		return
 	}
 
@@ -47,16 +48,22 @@ func (b *Bwhatsapp) reconnect(err error) {
 		Max:    5 * time.Minute,
 		Jitter: true,
 	}
+
 	for {
 		d := bf.Duration()
+
 		b.Log.Errorf("Connection failed, underlying error: %v", err)
 		b.Log.Infof("Waiting %s...", d)
+
 		time.Sleep(d)
+
 		b.Log.Info("Reconnecting...")
+
 		err := b.conn.Restore()
 		if err == nil {
 			bf.Reset()
 			b.startedAt = uint64(time.Now().Unix())
+
 			return
 		}
 	}
@@ -64,7 +71,7 @@ func (b *Bwhatsapp) reconnect(err error) {
 
 // HandleTextMessage sent from WhatsApp, relay it to the brige
 func (b *Bwhatsapp) HandleTextMessage(message whatsapp.TextMessage) {
-	if message.Info.FromMe { // || !strings.Contains(strings.ToLower(message.Text), "@echo") {
+	if message.Info.FromMe {
 		return
 	}
 	// whatsapp sends last messages to show context , cut them
@@ -72,13 +79,13 @@ func (b *Bwhatsapp) HandleTextMessage(message whatsapp.TextMessage) {
 		return
 	}
 
-	messageTime := time.Unix(int64(message.Info.Timestamp), 0) // TODO check how behaves between timezones
 	groupJID := message.Info.RemoteJid
-
 	senderJID := message.Info.SenderJid
+
 	if len(senderJID) == 0 {
-		// TODO workaround till https://github.com/Rhymen/go-whatsapp/issues/86 resolved
+		if message.Info.Source != nil && message.Info.Source.Participant != nil {
-		senderJID = *message.Info.Source.Participant
+			senderJID = *message.Info.Source.Participant
+		}
 	}
 
 	// translate sender's JID to the nicest username we can get
@@ -99,108 +106,275 @@ func (b *Bwhatsapp) HandleTextMessage(message whatsapp.TextMessage) {
 			if mention == "" {
 				mention = "someone"
 			}
+
 			message.Text = strings.Replace(message.Text, "@"+numberAndSuffix[0], "@"+mention, 1)
 		}
 	}
 
-	b.Log.Debugf("<= Sending message from %s on %s to gateway", senderJID, b.Account)
 	rmsg := config.Message{
-		UserID:    senderJID,
+		UserID:   senderJID,
-		Username:  senderName,
+		Username: senderName,
-		Text:      message.Text,
+		Text:     message.Text,
-		Timestamp: messageTime,
+		Channel:  groupJID,
-		Channel:   groupJID,
+		Account:  b.Account,
-		Account:   b.Account,
+		Protocol: b.Protocol,
-		Protocol:  b.Protocol,
+		Extra:    make(map[string][]interface{}),
-		Extra:     make(map[string][]interface{}),
 		//	ParentID: TODO, // TODO handle thread replies  // map from Info.QuotedMessageID string
-		//	Event     string    `json:"event"`
+		ID: message.Info.Id,
-		//	Gateway   string  // will be added during message processing
+	}
-		ID: message.Info.Id}
 
 	if avatarURL, exists := b.userAvatars[senderJID]; exists {
 		rmsg.Avatar = avatarURL
 	}
 
+	b.Log.Debugf("<= Sending message from %s on %s to gateway", senderJID, b.Account)
 	b.Log.Debugf("<= Message is %#v", rmsg)
+
 	b.Remote <- rmsg
 }
 
 // HandleImageMessage sent from WhatsApp, relay it to the brige
 func (b *Bwhatsapp) HandleImageMessage(message whatsapp.ImageMessage) {
-	if message.Info.FromMe { // || !strings.Contains(strings.ToLower(message.Text), "@echo") {
+	if message.Info.FromMe || message.Info.Timestamp < b.startedAt {
 		return
 	}
 
-	// whatsapp sends last messages to show context , cut them
-	if message.Info.Timestamp < b.startedAt {
-		return
-	}
-
-	messageTime := time.Unix(int64(message.Info.Timestamp), 0) // TODO check how behaves between timezones
-	groupJID := message.Info.RemoteJid
-
 	senderJID := message.Info.SenderJid
-	// if len(senderJid) == 0 {
+	if len(message.Info.SenderJid) == 0 && message.Info.Source != nil && message.Info.Source.Participant != nil {
-	//   // TODO workaround till https://github.com/Rhymen/go-whatsapp/issues/86 resolved
+		senderJID = *message.Info.Source.Participant
-	//   senderJid = *message.Info.Source.Participant
+	}
-	// }
 
-	// translate sender's Jid to the nicest username we can get
+	senderName := b.getSenderName(message.Info.SenderJid)
-	senderName := b.getSenderName(senderJID)
 	if senderName == "" {
 		senderName = "Someone" // don't expose telephone number
 	}
 
-	b.Log.Debugf("<= Sending message from %s on %s to gateway", senderJID, b.Account)
 	rmsg := config.Message{
-		UserID:    senderJID,
+		UserID:   senderJID,
-		Username:  senderName,
+		Username: senderName,
-		Timestamp: messageTime,
+		Channel:  message.Info.RemoteJid,
-		Channel:   groupJID,
+		Account:  b.Account,
-		Account:   b.Account,
+		Protocol: b.Protocol,
-		Protocol:  b.Protocol,
+		Extra:    make(map[string][]interface{}),
-		Extra:     make(map[string][]interface{}),
+		ID:       message.Info.Id,
-		//  ParentID: TODO,      // TODO handle thread replies  // map from Info.QuotedMessageID string
+	}
-		//  Event     string    `json:"event"`
-		//  Gateway   string     // will be added during message processing
-		ID: message.Info.Id}
 
 	if avatarURL, exists := b.userAvatars[senderJID]; exists {
 		rmsg.Avatar = avatarURL
 	}
 
-	// Download and unencrypt content
+	fileExt, err := mime.ExtensionsByType(message.Type)
-	data, err := message.Download()
 	if err != nil {
-		b.Log.Errorf("%v", err)
+		b.Log.Errorf("Mimetype detection error: %s", err)
+
 		return
 	}
 
-	// Get file extension by mimetype
+	// rename .jfif to .jpg https://github.com/42wim/matterbridge/issues/1292
-	fileExt, err := mime.ExtensionsByType(message.Type)
+	if fileExt[0] == ".jfif" {
-	if err != nil {
+		fileExt[0] = ".jpg"
-		b.Log.Errorf("%v", err)
+	}
-		return
+
+	// rename .jpe to .jpg https://github.com/42wim/matterbridge/issues/1463
+	if fileExt[0] == ".jpe" {
+		fileExt[0] = ".jpg"
 	}
 
 	filename := fmt.Sprintf("%v%v", message.Info.Id, fileExt[0])
 
-	b.Log.Debugf("<= Image downloaded and unencrypted")
+	b.Log.Debugf("Trying to download %s with type %s", filename, message.Type)
+
+	data, err := message.Download()
+	if err != nil {
+		b.Log.Errorf("Download image failed: %s", err)
+
+		return
+	}
 
 	// Move file to bridge storage
 	helper.HandleDownloadData(b.Log, &rmsg, filename, message.Caption, "", &data, b.General)
 
-	b.Log.Debugf("<= Image Message is %#v", rmsg)
+	b.Log.Debugf("<= Sending message from %s on %s to gateway", senderJID, b.Account)
+	b.Log.Debugf("<= Message is %#v", rmsg)
+
 	b.Remote <- rmsg
 }
 
-//func (b *Bwhatsapp) HandleVideoMessage(message whatsapp.VideoMessage) {
+// HandleVideoMessage downloads video messages
-//	fmt.Println(message) // TODO implement
+func (b *Bwhatsapp) HandleVideoMessage(message whatsapp.VideoMessage) {
-//}
+	if message.Info.FromMe || message.Info.Timestamp < b.startedAt {
-//
+		return
-//func (b *Bwhatsapp) HandleJsonMessage(message string) {
+	}
-//	fmt.Println(message) // TODO implement
+
-//}
+	senderJID := message.Info.SenderJid
-// TODO HandleRawMessage
+	if len(message.Info.SenderJid) == 0 && message.Info.Source != nil && message.Info.Source.Participant != nil {
-// TODO HandleAudioMessage
+		senderJID = *message.Info.Source.Participant
+	}
+
+	senderName := b.getSenderName(message.Info.SenderJid)
+	if senderName == "" {
+		senderName = "Someone" // don't expose telephone number
+	}
+
+	rmsg := config.Message{
+		UserID:   senderJID,
+		Username: senderName,
+		Channel:  message.Info.RemoteJid,
+		Account:  b.Account,
+		Protocol: b.Protocol,
+		Extra:    make(map[string][]interface{}),
+		ID:       message.Info.Id,
+	}
+
+	if avatarURL, exists := b.userAvatars[senderJID]; exists {
+		rmsg.Avatar = avatarURL
+	}
+
+	fileExt, err := mime.ExtensionsByType(message.Type)
+	if err != nil {
+		b.Log.Errorf("Mimetype detection error: %s", err)
+
+		return
+	}
+
+	if len(fileExt) == 0 {
+		fileExt = append(fileExt, ".mp4")
+	}
+
+	filename := fmt.Sprintf("%v%v", message.Info.Id, fileExt[0])
+
+	b.Log.Debugf("Trying to download %s with size %#v and type %s", filename, message.Length, message.Type)
+
+	data, err := message.Download()
+	if err != nil {
+		b.Log.Errorf("Download video failed: %s", err)
+
+		return
+	}
+
+	// Move file to bridge storage
+	helper.HandleDownloadData(b.Log, &rmsg, filename, message.Caption, "", &data, b.General)
+
+	b.Log.Debugf("<= Sending message from %s on %s to gateway", senderJID, b.Account)
+	b.Log.Debugf("<= Message is %#v", rmsg)
+
+	b.Remote <- rmsg
+}
+
+// HandleAudioMessage downloads audio messages
+func (b *Bwhatsapp) HandleAudioMessage(message whatsapp.AudioMessage) {
+	if message.Info.FromMe || message.Info.Timestamp < b.startedAt {
+		return
+	}
+
+	senderJID := message.Info.SenderJid
+	if len(message.Info.SenderJid) == 0 && message.Info.Source != nil && message.Info.Source.Participant != nil {
+		senderJID = *message.Info.Source.Participant
+	}
+
+	senderName := b.getSenderName(message.Info.SenderJid)
+	if senderName == "" {
+		senderName = "Someone" // don't expose telephone number
+	}
+
+	rmsg := config.Message{
+		UserID:   senderJID,
+		Username: senderName,
+		Channel:  message.Info.RemoteJid,
+		Account:  b.Account,
+		Protocol: b.Protocol,
+		Extra:    make(map[string][]interface{}),
+		ID:       message.Info.Id,
+	}
+
+	if avatarURL, exists := b.userAvatars[senderJID]; exists {
+		rmsg.Avatar = avatarURL
+	}
+
+	fileExt, err := mime.ExtensionsByType(message.Type)
+	if err != nil {
+		b.Log.Errorf("Mimetype detection error: %s", err)
+
+		return
+	}
+
+	if len(fileExt) == 0 {
+		fileExt = append(fileExt, ".ogg")
+	}
+
+	filename := fmt.Sprintf("%v%v", message.Info.Id, fileExt[0])
+
+	b.Log.Debugf("Trying to download %s with size %#v and type %s", filename, message.Length, message.Type)
+
+	data, err := message.Download()
+	if err != nil {
+		b.Log.Errorf("Download audio failed: %s", err)
+
+		return
+	}
+
+	// Move file to bridge storage
+	helper.HandleDownloadData(b.Log, &rmsg, filename, "audio message", "", &data, b.General)
+
+	b.Log.Debugf("<= Sending message from %s on %s to gateway", senderJID, b.Account)
+	b.Log.Debugf("<= Message is %#v", rmsg)
+
+	b.Remote <- rmsg
+}
+
+// HandleDocumentMessage downloads documents
+func (b *Bwhatsapp) HandleDocumentMessage(message whatsapp.DocumentMessage) {
+	if message.Info.FromMe || message.Info.Timestamp < b.startedAt {
+		return
+	}
+
+	senderJID := message.Info.SenderJid
+	if len(message.Info.SenderJid) == 0 && message.Info.Source != nil && message.Info.Source.Participant != nil {
+		senderJID = *message.Info.Source.Participant
+	}
+
+	senderName := b.getSenderName(message.Info.SenderJid)
+	if senderName == "" {
+		senderName = "Someone" // don't expose telephone number
+	}
+
+	rmsg := config.Message{
+		UserID:   senderJID,
+		Username: senderName,
+		Channel:  message.Info.RemoteJid,
+		Account:  b.Account,
+		Protocol: b.Protocol,
+		Extra:    make(map[string][]interface{}),
+		ID:       message.Info.Id,
+	}
+
+	if avatarURL, exists := b.userAvatars[senderJID]; exists {
+		rmsg.Avatar = avatarURL
+	}
+
+	fileExt, err := mime.ExtensionsByType(message.Type)
+	if err != nil {
+		b.Log.Errorf("Mimetype detection error: %s", err)
+
+		return
+	}
+
+	filename := fmt.Sprintf("%v", message.FileName)
+
+	b.Log.Debugf("Trying to download %s with extension %s and type %s", filename, fileExt, message.Type)
+
+	data, err := message.Download()
+	if err != nil {
+		b.Log.Errorf("Download document message failed: %s", err)
+
+		return
+	}
+
+	// Move file to bridge storage
+	helper.HandleDownloadData(b.Log, &rmsg, filename, "document", "", &data, b.General)
+
+	b.Log.Debugf("<= Sending message from %s on %s to gateway", senderJID, b.Account)
+	b.Log.Debugf("<= Message is %#v", rmsg)
+
+	b.Remote <- rmsg
+}

bridge/whatsapp/helpers.go

@@ -6,22 +6,24 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"strings"
 
 	qrcodeTerminal "github.com/Baozisoftware/qrcode-terminal-go"
 	"github.com/Rhymen/go-whatsapp"
 )
 
 type ProfilePicInfo struct {
-	URL string `json:"eurl"`
+	URL    string `json:"eurl"`
-	Tag string `json:"tag"`
+	Tag    string `json:"tag"`
-
+	Status int16  `json:"status"`
-	Status int16 `json:"status"`
 }
 
 func qrFromTerminal(invert bool) chan string {
 	qr := make(chan string)
+
 	go func() {
 		terminal := qrcodeTerminal.New()
+
 		if invert {
 			terminal = qrcodeTerminal.New2(qrcodeTerminal.ConsoleColors.BrightWhite, qrcodeTerminal.ConsoleColors.BrightBlack, qrcodeTerminal.QRCodeRecoveryLevels.Medium)
 		}
@@ -44,13 +46,12 @@ func (b *Bwhatsapp) readSession() (whatsapp.Session, error) {
 	if err != nil {
 		return session, err
 	}
+
 	defer file.Close()
+
 	decoder := gob.NewDecoder(file)
-	err = decoder.Decode(&session)
+
-	if err != nil {
+	return session, decoder.Decode(&session)
-		return session, err
-	}
-	return session, nil
 }
 
 func (b *Bwhatsapp) writeSession(session whatsapp.Session) error {
@@ -65,11 +66,31 @@ func (b *Bwhatsapp) writeSession(session whatsapp.Session) error {
 	if err != nil {
 		return err
 	}
-	defer file.Close()
-	encoder := gob.NewEncoder(file)
-	err = encoder.Encode(session)
 
-	return err
+	defer file.Close()
+
+	encoder := gob.NewEncoder(file)
+
+	return encoder.Encode(session)
+}
+
+func (b *Bwhatsapp) restoreSession() (*whatsapp.Session, error) {
+	session, err := b.readSession()
+	if err != nil {
+		b.Log.Warn(err.Error())
+	}
+
+	b.Log.Debugln("Restoring WhatsApp session..")
+
+	session, err = b.conn.RestoreWithSession(session)
+	if err != nil {
+		// restore session connection timed out (I couldn't get over it without logging in again)
+		return nil, errors.New("failed to restore session: " + err.Error())
+	}
+
+	b.Log.Debugln("Session restored successfully!")
+
+	return &session, nil
 }
 
 func (b *Bwhatsapp) getSenderName(senderJid string) string {
@@ -114,6 +135,7 @@ func (b *Bwhatsapp) getSenderNotify(senderJid string) string {
 	if sender, exists := b.users[senderJid]; exists {
 		return sender.Notify
 	}
+
 	return ""
 }
 
@@ -122,11 +144,20 @@ func (b *Bwhatsapp) GetProfilePicThumb(jid string) (*ProfilePicInfo, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to get avatar: %v", err)
 	}
+
 	content := <-data
 	info := &ProfilePicInfo{}
+
 	err = json.Unmarshal([]byte(content), info)
 	if err != nil {
 		return info, fmt.Errorf("failed to unmarshal avatar info: %v", err)
 	}
+
 	return info, nil
 }
+
+func isGroupJid(identifier string) bool {
+	return strings.HasSuffix(identifier, "@g.us") ||
+		strings.HasSuffix(identifier, "@temp") ||
+		strings.HasSuffix(identifier, "@broadcast")
+}

bridge/whatsapp/whatsapp.go

@@ -28,7 +28,6 @@ const (
 type Bwhatsapp struct {
 	*bridge.Config
 
-	// https://github.com/Rhymen/go-whatsapp/blob/c31092027237441cffba1b9cb148eadf7c83c3d2/session.go#L18-L21
 	session   *whatsapp.Session
 	conn      *whatsapp.Conn
 	startedAt uint64
@@ -40,6 +39,7 @@ type Bwhatsapp struct {
 // New Create a new WhatsApp bridge. This will be called for each [whatsapp.<server>] entry you have in the config file
 func New(cfg *bridge.Config) bridge.Bridger {
 	number := cfg.GetString(cfgNumber)
+
 	if number == "" {
 		cfg.Log.Fatalf("Missing configuration for WhatsApp bridge: Number")
 	}
@@ -50,24 +50,19 @@ func New(cfg *bridge.Config) bridge.Bridger {
 		users:       make(map[string]whatsapp.Contact),
 		userAvatars: make(map[string]string),
 	}
+
 	return b
 }
 
 // Connect to WhatsApp. Required implementation of the Bridger interface
-// https://github.com/42wim/matterbridge/blob/2cfd880cdb0df29771bf8f31df8d990ab897889d/bridge/bridge.go#L11-L16
 func (b *Bwhatsapp) Connect() error {
-	b.RLock() // TODO do we need locking for Whatsapp?
-	defer b.RUnlock()
-
 	number := b.GetString(cfgNumber)
 	if number == "" {
-		return errors.New("WhatsApp's telephone Number need to be configured")
+		return errors.New("whatsapp's telephone number need to be configured")
 	}
 
-	// https://github.com/Rhymen/go-whatsapp#creating-a-connection
 	b.Log.Debugln("Connecting to WhatsApp..")
 	conn, err := whatsapp.NewConn(20 * time.Second)
-	conn.SetClientVersion(0, 4, 2080)
 	if err != nil {
 		return errors.New("failed to connect to WhatsApp: " + err.Error())
 	}
@@ -78,35 +73,18 @@ func (b *Bwhatsapp) Connect() error {
 	b.Log.Debugln("WhatsApp connection successful")
 
 	// load existing session in order to keep it between restarts
-	if b.session == nil {
+	b.session, err = b.restoreSession()
-		var session whatsapp.Session
+	if err != nil {
-		session, err = b.readSession()
+		b.Log.Warn(err.Error())
-
-		if err == nil {
-			b.Log.Debugln("Restoring WhatsApp session..")
-
-			// https://github.com/Rhymen/go-whatsapp#restore
-			session, err = b.conn.RestoreWithSession(session)
-			if err != nil {
-				// TODO return or continue to normal login?
-				// restore session connection timed out (I couldn't get over it without logging in again)
-				return errors.New("failed to restore session: " + err.Error())
-			}
-
-			b.session = &session
-			b.Log.Debugln("Session restored successfully!")
-		} else {
-			b.Log.Warn(err.Error())
-		}
 	}
 
 	// login to a new session
 	if b.session == nil {
-		err = b.Login()
+		if err = b.Login(); err != nil {
-		if err != nil {
 			return err
 		}
 	}
+
 	b.startedAt = uint64(time.Now().Unix())
 
 	_, err = b.conn.Contacts()
@@ -114,6 +92,13 @@ func (b *Bwhatsapp) Connect() error {
 		return fmt.Errorf("error on update of contacts: %v", err)
 	}
 
+	// see https://github.com/Rhymen/go-whatsapp/issues/137#issuecomment-480316013
+	for len(b.conn.Store.Contacts) == 0 {
+		b.conn.Contacts() // nolint:errcheck
+
+		<-time.After(1 * time.Second)
+	}
+
 	// map all the users
 	for id, contact := range b.conn.Store.Contacts {
 		if !isGroupJid(id) && id != "status@broadcast" {
@@ -130,12 +115,13 @@ func (b *Bwhatsapp) Connect() error {
 			info, err := b.GetProfilePicThumb(jid)
 			if err != nil {
 				b.Log.Warnf("Could not get profile photo of %s: %v", jid, err)
-
 			} else {
-				// TODO any race conditions here?
+				b.Lock()
 				b.userAvatars[jid] = info.URL
+				b.Unlock()
 			}
 		}
+
 		b.Log.Debug("Finished getting avatars..")
 	}()
 
@@ -152,8 +138,10 @@ func (b *Bwhatsapp) Login() error {
 	session, err := b.conn.Login(qrChan)
 	if err != nil {
 		b.Log.Warnln("Failed to log in:", err)
+
 		return err
 	}
+
 	b.session = &session
 
 	b.Log.Infof("Logged into session: %#v", session)
@@ -164,74 +152,62 @@ func (b *Bwhatsapp) Login() error {
 		fmt.Fprintf(os.Stderr, "error saving session: %v\n", err)
 	}
 
-	// TODO change connection strings to configured ones longClientName:"github.com/rhymen/go-whatsapp", shortClientName:"go-whatsapp"}" prefix=whatsapp
-	// TODO get also a nice logo
-
-	// TODO notification about unplugged and dead battery
-	// conn.Info: Wid, Pushname, Connected, Battery, Plugged
-
 	return nil
 }
 
 // Disconnect is called while reconnecting to the bridge
-// TODO 42wim Documentation would be helpful on when reconnects happen and what should be done in this function
 // Required implementation of the Bridger interface
-// https://github.com/42wim/matterbridge/blob/2cfd880cdb0df29771bf8f31df8d990ab897889d/bridge/bridge.go#L11-L16
 func (b *Bwhatsapp) Disconnect() error {
 	// We could Logout, but that would close the session completely and would require a new QR code scan
 	// https://github.com/Rhymen/go-whatsapp/blob/c31092027237441cffba1b9cb148eadf7c83c3d2/session.go#L377-L381
 	return nil
 }
 
-func isGroupJid(identifier string) bool {
-	return strings.HasSuffix(identifier, "@g.us") || strings.HasSuffix(identifier, "@temp")
-}
-
 // JoinChannel Join a WhatsApp group specified in gateway config as channel='number-id@g.us' or channel='Channel name'
 // Required implementation of the Bridger interface
 // https://github.com/42wim/matterbridge/blob/2cfd880cdb0df29771bf8f31df8d990ab897889d/bridge/bridge.go#L11-L16
 func (b *Bwhatsapp) JoinChannel(channel config.ChannelInfo) error {
 	byJid := isGroupJid(channel.Name)
 
+	// see https://github.com/Rhymen/go-whatsapp/issues/137#issuecomment-480316013
+	for len(b.conn.Store.Contacts) == 0 {
+		b.conn.Contacts() // nolint:errcheck
+		<-time.After(1 * time.Second)
+	}
+
 	// verify if we are member of the given group
 	if byJid {
 		// channel.Name specifies static group jID, not the name
 		if _, exists := b.conn.Store.Contacts[channel.Name]; !exists {
 			return fmt.Errorf("account doesn't belong to group with jid %s", channel.Name)
 		}
-	} else {
-		// channel.Name specifies group name that might change, warn about it
-		var jids []string
-		for id, contact := range b.conn.Store.Contacts {
-			if isGroupJid(id) && contact.Name == channel.Name {
-				jids = append(jids, id)
-			}
-		}
 
-		switch len(jids) {
+		return nil
-		case 0:
+	}
-			// didn't match any group - print out possibilites
-			// TODO sort
-			// copy b;
-			//sort.Slice(people, func(i, j int) bool {
-			//	return people[i].Age > people[j].Age
-			//})
-			for id, contact := range b.conn.Store.Contacts {
-				if isGroupJid(id) {
-					b.Log.Infof("%s %s", contact.Jid, contact.Name)
-				}
-			}
-			return fmt.Errorf("please specify group's JID from the list above instead of the name '%s'", channel.Name)
 
-		case 1:
+	// channel.Name specifies group name that might change, warn about it
-			return fmt.Errorf("group name might change. Please configure gateway with channel=\"%v\" instead of channel=\"%v\"", jids[0], channel.Name)
+	var jids []string
-
+	for id, contact := range b.conn.Store.Contacts {
-		default:
+		if isGroupJid(id) && contact.Name == channel.Name {
-			return fmt.Errorf("there is more than one group with name '%s'. Please specify one of JIDs as channel name: %v", channel.Name, jids)
+			jids = append(jids, id)
 		}
 	}
 
-	return nil
+	switch len(jids) {
+	case 0:
+		// didn't match any group - print out possibilites
+		for id, contact := range b.conn.Store.Contacts {
+			if isGroupJid(id) {
+				b.Log.Infof("%s %s", contact.Jid, contact.Name)
+			}
+		}
+
+		return fmt.Errorf("please specify group's JID from the list above instead of the name '%s'", channel.Name)
+	case 1:
+		return fmt.Errorf("group name might change. Please configure gateway with channel=\"%v\" instead of channel=\"%v\"", jids[0], channel.Name)
+	default:
+		return fmt.Errorf("there is more than one group with name '%s'. Please specify one of JIDs as channel name: %v", channel.Name, jids)
+	}
 }
 
 // Post a document message from the bridge to WhatsApp
@@ -305,14 +281,12 @@ func (b *Bwhatsapp) Send(msg config.Message) (string, error) {
 		if msg.ID == "" {
 			// No message ID in case action is executed on a message sent before the bridge was started
 			// and then the bridge cache doesn't have this message ID mapped
-
-			// TODO 42wim Doesn't the app get clogged with a ton of IDs after some time of running?
-			// WhatsApp allows to set any ID so in that case we could use external IDs and don't do mapping
-			// but external IDs are not set
 			return "", nil
 		}
-		// TODO delete message on WhatsApp https://github.com/Rhymen/go-whatsapp/issues/100
+
-		return "", nil
+		_, err := b.conn.RevokeMessage(msg.Channel, msg.ID, true)
+
+		return "", err
 	}
 
 	// Edit message
@@ -320,7 +294,6 @@ func (b *Bwhatsapp) Send(msg config.Message) (string, error) {
 		b.Log.Debugf("updating message with id %s", msg.ID)
 
 		msg.Text += " (edited)"
-		// TODO handle edit as a message reply with updated text
 	}
 
 	// Handle Upload a file
@@ -350,16 +323,7 @@ func (b *Bwhatsapp) Send(msg config.Message) (string, error) {
 
 	b.Log.Debugf("=> Sending %#v", msg)
 
-	// create message ID
+	return b.conn.Send(message)
-	// TODO follow and act if https://github.com/Rhymen/go-whatsapp/issues/101 implemented
-	idBytes := make([]byte, 10)
-	if _, err := rand.Read(idBytes); err != nil {
-		b.Log.Warn(err.Error())
-	}
-	message.Info.Id = strings.ToUpper(hex.EncodeToString(idBytes))
-	_, err := b.conn.Send(message)
-
-	return message.Info.Id, err
 }
 
 // TODO do we want that? to allow login with QR code from a bridged channel? https://github.com/tulir/mautrix-whatsapp/blob/513eb18e2d59bada0dd515ee1abaaf38a3bfe3d5/commands.go#L76

bridge/xmpp/xmpp.go

@@ -1,8 +1,12 @@
 package bxmpp
 
 import (
+	"bytes"
 	"crypto/tls"
+	"encoding/json"
 	"fmt"
+	"net/http"
+	"net/url"
 	"strings"
 	"sync"
 	"time"
@@ -86,14 +90,21 @@ func (b *Bxmpp) Send(msg config.Message) (string, error) {
 	}
 
 	// Upload a file (in XMPP case send the upload URL because XMPP has no native upload support).
+	var err error
 	if msg.Extra != nil {
 		for _, rmsg := range helper.HandleExtra(&msg, b.General) {
 			b.Log.Debugf("=> Sending attachement message %#v", rmsg)
-			if _, err := b.xc.Send(xmpp.Chat{
+			if b.GetString("WebhookURL") != "" {
-				Type:   "groupchat",
+				err = b.postSlackCompatibleWebhook(msg)
-				Remote: rmsg.Channel + "@" + b.GetString("Muc"),
+			} else {
-				Text:   rmsg.Username + rmsg.Text,
+				_, err = b.xc.Send(xmpp.Chat{
-			}); err != nil {
+					Type:   "groupchat",
+					Remote: rmsg.Channel + "@" + b.GetString("Muc"),
+					Text:   rmsg.Username + rmsg.Text,
+				})
+			}
+
+			if err != nil {
 				b.Log.WithError(err).Error("Unable to send message with share URL.")
 			}
 		}
@@ -102,13 +113,23 @@ func (b *Bxmpp) Send(msg config.Message) (string, error) {
 		}
 	}
 
+	if b.GetString("WebhookURL") != "" {
+		b.Log.Debugf("Sending message using Webhook")
+		err := b.postSlackCompatibleWebhook(msg)
+		if err != nil {
+			b.Log.Errorf("Failed to send message using webhook: %s", err)
+			return "", err
+		}
+
+		return "", nil
+	}
+
+	// Post normal message.
 	var msgReplaceID string
 	msgID := xid.New().String()
 	if msg.ID != "" {
-		msgID = msg.ID
 		msgReplaceID = msg.ID
 	}
-	// Post normal message.
 	b.Log.Debugf("=> Sending message %#v", msg)
 	if _, err := b.xc.Send(xmpp.Chat{
 		Type:      "groupchat",
@@ -122,12 +143,46 @@ func (b *Bxmpp) Send(msg config.Message) (string, error) {
 	return msgID, nil
 }
 
-func (b *Bxmpp) createXMPP() error {
+func (b *Bxmpp) postSlackCompatibleWebhook(msg config.Message) error {
-	if !strings.Contains(b.GetString("Jid"), "@") {
+	type XMPPWebhook struct {
-		return fmt.Errorf("the Jid %s doesn't contain an @", b.GetString("Jid"))
+		Username string `json:"username"`
+		Text     string `json:"text"`
 	}
+	webhookBody, err := json.Marshal(XMPPWebhook{
+		Username: msg.Username,
+		Text:     msg.Text,
+	})
+	if err != nil {
+		b.Log.Errorf("Failed to marshal webhook: %s", err)
+		return err
+	}
+
+	resp, err := http.Post(b.GetString("WebhookURL")+"/"+url.QueryEscape(msg.Channel), "application/json", bytes.NewReader(webhookBody))
+	if err != nil {
+		b.Log.Errorf("Failed to POST webhook: %s", err)
+		return err
+	}
+
+	resp.Body.Close()
+	return nil
+}
+
+func (b *Bxmpp) createXMPP() error {
+	var serverName string
+	switch {
+	case !b.GetBool("Anonymous"):
+		if !strings.Contains(b.GetString("Jid"), "@") {
+			return fmt.Errorf("the Jid %s doesn't contain an @", b.GetString("Jid"))
+		}
+		serverName = strings.Split(b.GetString("Jid"), "@")[1]
+	case !strings.Contains(b.GetString("Server"), ":"):
+		serverName = strings.Split(b.GetString("Server"), ":")[0]
+	default:
+		serverName = b.GetString("Server")
+	}
+
 	tc := &tls.Config{
-		ServerName:         strings.Split(b.GetString("Jid"), "@")[1],
+		ServerName:         serverName,
 		InsecureSkipVerify: b.GetBool("SkipTLSVerify"), // nolint: gosec
 	}
 
@@ -138,14 +193,14 @@ func (b *Bxmpp) createXMPP() error {
 		User:                         b.GetString("Jid"),
 		Password:                     b.GetString("Password"),
 		NoTLS:                        true,
-		StartTLS:                     true,
+		StartTLS:                     !b.GetBool("NoTLS"),
 		TLSConfig:                    tc,
 		Debug:                        b.GetBool("debug"),
 		Session:                      true,
 		Status:                       "",
 		StatusMessage:                "",
 		Resource:                     "",
-		InsecureAllowUnencryptedAuth: false,
+		InsecureAllowUnencryptedAuth: b.GetBool("NoTLS"),
 	}
 	var err error
 	b.xc, err = options.NewClient()
@@ -228,7 +283,13 @@ func (b *Bxmpp) handleXMPP() error {
 	for {
 		m, err := b.xc.Recv()
 		if err != nil {
-			return err
+			// An error together with AvatarData is non-fatal
+			switch m.(type) {
+			case xmpp.AvatarData:
+				continue
+			default:
+				return err
+			}
 		}
 
 		switch v := m.(type) {
@@ -339,7 +400,7 @@ func (b *Bxmpp) handleUploadFile(msg *config.Message) error {
 
 func (b *Bxmpp) parseNick(remote string) string {
 	s := strings.Split(remote, "@")
-	if len(s) > 0 {
+	if len(s) > 1 {
 		s = strings.Split(s[1], "/")
 		if len(s) == 2 {
 			return s[1] // nick
@@ -378,6 +439,11 @@ func (b *Bxmpp) skipMessage(message xmpp.Chat) bool {
 		return true
 	}
 
+	// Ignore messages posted by our webhook
+	if b.GetString("WebhookURL") != "" && strings.Contains(message.ID, "webhookbot") {
+		return true
+	}
+
 	// skip delayed messages
 	return !message.Stamp.IsZero() && time.Since(message.Stamp).Minutes() > 5
 }

bridge/zulip/zulip.go

@@ -2,6 +2,7 @@ package bzulip
 
 import (
 	"encoding/json"
+	"fmt"
 	"io/ioutil"
 	"strconv"
 	"strings"
@@ -11,6 +12,7 @@ import (
 	"github.com/42wim/matterbridge/bridge"
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/42wim/matterbridge/bridge/helper"
+	"github.com/42wim/matterbridge/version"
 	gzb "github.com/matterbridge/gozulipbot"
 )
 
@@ -27,7 +29,7 @@ func New(cfg *bridge.Config) bridge.Bridger {
 }
 
 func (b *Bzulip) Connect() error {
-	bot := gzb.Bot{APIKey: b.GetString("token"), APIURL: b.GetString("server") + "/api/v1/", Email: b.GetString("login")}
+	bot := gzb.Bot{APIKey: b.GetString("token"), APIURL: b.GetString("server") + "/api/v1/", Email: b.GetString("login"), UserAgent: fmt.Sprintf("matterbridge/%s", version.Release)}
 	bot.Init()
 	q, err := bot.RegisterAll()
 	b.q = q
@@ -125,6 +127,7 @@ func (b *Bzulip) handleQueue() error {
 			b.Log.Debug("heartbeat received.")
 		default:
 			b.Log.Debugf("receiving error: %#v", err)
+			time.Sleep(time.Second * 10)
 		}
 		if err != nil {
 			continue
@@ -135,19 +138,25 @@ func (b *Bzulip) handleQueue() error {
 			if m.SenderEmail == b.GetString("login") {
 				continue
 			}
+
+			avatarURL := m.AvatarURL
+			if !strings.HasPrefix(avatarURL, "http") {
+				avatarURL = b.GetString("server") + avatarURL
+			}
+
 			rmsg := config.Message{
 				Username: m.SenderFullName,
 				Text:     m.Content,
 				Channel:  b.getChannel(m.StreamID) + "/topic:" + m.Subject,
 				Account:  b.Account,
 				UserID:   strconv.Itoa(m.SenderID),
-				Avatar:   m.AvatarURL,
+				Avatar:   avatarURL,
 			}
 			b.Log.Debugf("<= Sending message from %s on %s to gateway", rmsg.Username, b.Account)
 			b.Log.Debugf("<= Message is %#v", rmsg)
 			b.Remote <- rmsg
-			b.q.LastEventID = m.ID
 		}
+
 		time.Sleep(time.Second * 3)
 	}
 }

docker/arm/Dockerfile

@@ -1,9 +1,10 @@
 FROM alpine:edge as certs
 RUN apk --update add ca-certificates
+ARG VERSION=1.22.3
+ADD https://github.com/42wim/matterbridge/releases/download/v${VERSION}/matterbridge-${VERSION}-linux-arm64 /bin/matterbridge
+RUN chmod +x /bin/matterbridge
 
 FROM scratch
-ARG VERSION=1.12.3
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
-ADD https://github.com/42wim/matterbridge/releases/download/v${VERSION}/matterbridge-linux-arm /bin/matterbridge
+COPY --from=certs /bin/matterbridge /bin/matterbridge
-RUN chmod +x /bin/matterbridge
 ENTRYPOINT ["/bin/matterbridge"]

gateway/bridgemap/bmumble.go

@@ -0,0 +1,11 @@
+// +build !nomumble
+
+package bridgemap
+
+import (
+	bmumble "github.com/42wim/matterbridge/bridge/mumble"
+)
+
+func init() {
+	FullMap["mumble"] = bmumble.New
+}

gateway/bridgemap/bvk.go

@@ -0,0 +1,11 @@
+// +build !novk
+
+package bridgemap
+
+import (
+	bvk "github.com/42wim/matterbridge/bridge/vk"
+)
+
+func init() {
+	FullMap["vk"] = bvk.New
+}

gateway/gateway.go

@@ -1,6 +1,7 @@
 package gateway
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
 	"regexp"
@@ -13,7 +14,7 @@ import (
 	"github.com/d5/tengo/v2"
 	"github.com/d5/tengo/v2/stdlib"
 	lru "github.com/hashicorp/golang-lru"
-	"github.com/matterbridge/emoji"
+	"github.com/kyokomi/emoji/v2"
 	"github.com/sirupsen/logrus"
 )
 
@@ -126,7 +127,7 @@ func (gw *Gateway) AddConfig(cfg *config.Gateway) error {
 		gw.logger.Errorf("mapChannels() failed: %s", err)
 	}
 	for _, br := range append(gw.MyConfig.In, append(gw.MyConfig.InOut, gw.MyConfig.Out...)...) {
-		br := br //scopelint
+		br := br // scopelint
 		err := gw.AddBridge(&br)
 		if err != nil {
 			return err
@@ -336,20 +337,21 @@ func (gw *Gateway) modifyUsername(msg *config.Message, dest *bridge.Bridge) stri
 			}
 			i++
 		}
-		nick = strings.Replace(nick, "{NOPINGNICK}", msg.Username[:i]+"​"+msg.Username[i:], -1)
+		nick = strings.ReplaceAll(nick, "{NOPINGNICK}", msg.Username[:i]+"\u200b"+msg.Username[i:])
 	}
 
-	nick = strings.Replace(nick, "{BRIDGE}", br.Name, -1)
+	nick = strings.ReplaceAll(nick, "{BRIDGE}", br.Name)
-	nick = strings.Replace(nick, "{PROTOCOL}", br.Protocol, -1)
+	nick = strings.ReplaceAll(nick, "{PROTOCOL}", br.Protocol)
-	nick = strings.Replace(nick, "{GATEWAY}", gw.Name, -1)
+	nick = strings.ReplaceAll(nick, "{GATEWAY}", gw.Name)
-	nick = strings.Replace(nick, "{LABEL}", br.GetString("Label"), -1)
+	nick = strings.ReplaceAll(nick, "{LABEL}", br.GetString("Label"))
-	nick = strings.Replace(nick, "{NICK}", msg.Username, -1)
+	nick = strings.ReplaceAll(nick, "{NICK}", msg.Username)
-	nick = strings.Replace(nick, "{CHANNEL}", msg.Channel, -1)
+	nick = strings.ReplaceAll(nick, "{USERID}", msg.UserID)
+	nick = strings.ReplaceAll(nick, "{CHANNEL}", msg.Channel)
 	tengoNick, err := gw.modifyUsernameTengo(msg, br)
 	if err != nil {
 		gw.logger.Errorf("modifyUsernameTengo error: %s", err)
 	}
-	nick = strings.Replace(nick, "{TENGO}", tengoNick, -1) //nolint:gocritic
+	nick = strings.ReplaceAll(nick, "{TENGO}", tengoNick)
 	return nick
 }
 
@@ -363,14 +365,28 @@ func (gw *Gateway) modifyAvatar(msg *config.Message, dest *bridge.Bridge) string
 }
 
 func (gw *Gateway) modifyMessage(msg *config.Message) {
-	if err := modifyMessageTengo(gw.BridgeValues().General.TengoModifyMessage, msg); err != nil {
+	if gw.BridgeValues().General.TengoModifyMessage != "" {
+		gw.logger.Warnf("General TengoModifyMessage=%s is deprecated and will be removed in v1.20.0, please move to Tengo InMessage=%s", gw.BridgeValues().General.TengoModifyMessage, gw.BridgeValues().General.TengoModifyMessage)
+	}
+
+	if err := modifyInMessageTengo(gw.BridgeValues().General.TengoModifyMessage, msg); err != nil {
 		gw.logger.Errorf("TengoModifyMessage failed: %s", err)
 	}
-	if err := modifyMessageTengo(gw.BridgeValues().Tengo.Message, msg); err != nil {
+
+	inMessage := gw.BridgeValues().Tengo.InMessage
+	if inMessage == "" {
+		inMessage = gw.BridgeValues().Tengo.Message
+		if inMessage != "" {
+			gw.logger.Warnf("Tengo Message=%s is deprecated and will be removed in v1.20.0, please move to Tengo InMessage=%s", inMessage, inMessage)
+		}
+	}
+
+	if err := modifyInMessageTengo(inMessage, msg); err != nil {
 		gw.logger.Errorf("Tengo.Message failed: %s", err)
 	}
 
 	// replace :emoji: to unicode
+	emoji.ReplacePadding = ""
 	msg.Text = emoji.Sprint(msg.Text)
 
 	br := gw.Bridges[msg.Account]
@@ -416,9 +432,15 @@ func (gw *Gateway) SendMessage(
 		}
 	}
 
+	// Only send irc notices to irc
+	if msg.Event == config.EventNoticeIRC && dest.Protocol != "irc" {
+		return "", nil
+	}
+
 	// Too noisy to log like other events
+	debugSendMessage := ""
 	if msg.Event != config.EventUserTyping {
-		gw.logger.Debugf("=> Sending %#v from %s (%s) to %s (%s)", msg, msg.Account, rmsg.Channel, dest.Account, channel.Name)
+		debugSendMessage = fmt.Sprintf("=> Sending %#v from %s (%s) to %s (%s)", msg, msg.Account, rmsg.Channel, dest.Account, channel.Name)
 	}
 
 	msg.Channel = channel.Name
@@ -438,22 +460,34 @@ func (gw *Gateway) SendMessage(
 	}
 
 	// if the parentID is still empty and we have a parentID set in the original message
-	// this means that we didn't find it in the cache so set it "msg-parent-not-found"
+	// this means that we didn't find it in the cache so set it to a "msg-parent-not-found" constant
 	if msg.ParentID == "" && rmsg.ParentID != "" {
-		msg.ParentID = "msg-parent-not-found"
+		msg.ParentID = config.ParentIDNotFound
 	}
 
-	err := gw.modifySendMessageTengo(rmsg, &msg, dest)
+	drop, err := gw.modifyOutMessageTengo(rmsg, &msg, dest)
 	if err != nil {
 		gw.logger.Errorf("modifySendMessageTengo: %s", err)
 	}
 
+	if drop {
+		gw.logger.Debugf("=> Tengo dropping %#v from %s (%s) to %s (%s)", msg, msg.Account, rmsg.Channel, dest.Account, channel.Name)
+		return "", nil
+	}
+
+	if debugSendMessage != "" {
+		gw.logger.Debug(debugSendMessage)
+	}
 	// if we are using mattermost plugin account, send messages to MattermostPlugin channel
 	// that can be picked up by the mattermost matterbridge plugin
 	if dest.Account == "mattermost.plugin" {
 		gw.Router.MattermostPlugin <- msg
 	}
 
+	defer func(t time.Time) {
+		gw.logger.Debugf("=> Send from %s (%s) to %s (%s) took %s", msg.Account, rmsg.Channel, dest.Account, channel.Name, time.Since(t))
+	}(time.Now())
+
 	mID, err := dest.Send(msg)
 	if err != nil {
 		return mID, err
@@ -463,7 +497,7 @@ func (gw *Gateway) SendMessage(
 	if mID != "" {
 		gw.logger.Debugf("mID %s: %s", dest.Account, mID)
 		return mID, nil
-		//brMsgIDs = append(brMsgIDs, &BrMsgID{dest, dest.Protocol + " " + mID, channel.ID})
+		// brMsgIDs = append(brMsgIDs, &BrMsgID{dest, dest.Protocol + " " + mID, channel.ID})
 	}
 	return "", nil
 }
@@ -505,7 +539,7 @@ func getProtocol(msg *config.Message) string {
 	return p[0]
 }
 
-func modifyMessageTengo(filename string, msg *config.Message) error {
+func modifyInMessageTengo(filename string, msg *config.Message) error {
 	if filename == "" {
 		return nil
 	}
@@ -517,6 +551,7 @@ func modifyMessageTengo(filename string, msg *config.Message) error {
 	s.SetImports(stdlib.GetModuleMap(stdlib.AllModuleNames()...))
 	_ = s.Add("msgText", msg.Text)
 	_ = s.Add("msgUsername", msg.Username)
+	_ = s.Add("msgUserID", msg.UserID)
 	_ = s.Add("msgAccount", msg.Account)
 	_ = s.Add("msgChannel", msg.Channel)
 	c, err := s.Compile()
@@ -545,6 +580,7 @@ func (gw *Gateway) modifyUsernameTengo(msg *config.Message, br *bridge.Bridge) (
 	_ = s.Add("result", "")
 	_ = s.Add("msgText", msg.Text)
 	_ = s.Add("msgUsername", msg.Username)
+	_ = s.Add("msgUserID", msg.UserID)
 	_ = s.Add("nick", msg.Username)
 	_ = s.Add("msgAccount", msg.Account)
 	_ = s.Add("msgChannel", msg.Channel)
@@ -564,22 +600,28 @@ func (gw *Gateway) modifyUsernameTengo(msg *config.Message, br *bridge.Bridge) (
 	return c.Get("result").String(), nil
 }
 
-func (gw *Gateway) modifySendMessageTengo(origmsg *config.Message, msg *config.Message, br *bridge.Bridge) error {
+func (gw *Gateway) modifyOutMessageTengo(origmsg *config.Message, msg *config.Message, br *bridge.Bridge) (bool, error) {
 	filename := gw.BridgeValues().Tengo.OutMessage
-	var res []byte
+	var (
-	var err error
+		res  []byte
+		err  error
+		drop bool
+	)
+
 	if filename == "" {
 		res, err = internal.Asset("tengo/outmessage.tengo")
 		if err != nil {
-			return err
+			return drop, err
 		}
 	} else {
 		res, err = ioutil.ReadFile(filename)
 		if err != nil {
-			return err
+			return drop, err
 		}
 	}
+
 	s := tengo.NewScript(res)
+
 	s.SetImports(stdlib.GetModuleMap(stdlib.AllModuleNames()...))
 	_ = s.Add("inAccount", origmsg.Account)
 	_ = s.Add("inProtocol", origmsg.Protocol)
@@ -593,14 +635,20 @@ func (gw *Gateway) modifySendMessageTengo(origmsg *config.Message, msg *config.M
 	_ = s.Add("outEvent", msg.Event)
 	_ = s.Add("msgText", msg.Text)
 	_ = s.Add("msgUsername", msg.Username)
+	_ = s.Add("msgUserID", msg.UserID)
+	_ = s.Add("msgDrop", drop)
 	c, err := s.Compile()
 	if err != nil {
-		return err
+		return drop, err
 	}
+
 	if err := c.Run(); err != nil {
-		return err
+		return drop, err
 	}
+
+	drop = c.Get("msgDrop").Bool()
 	msg.Text = c.Get("msgText").String()
 	msg.Username = c.Get("msgUsername").String()
-	return nil
+
+	return drop, nil
 }

gateway/gateway_test.go

@@ -533,7 +533,7 @@ func (s *ignoreTestSuite) TestIgnoreNicks() {
 func BenchmarkTengo(b *testing.B) {
 	msg := &config.Message{Username: "user", Text: "blah testing", Account: "protocol.account", Channel: "mychannel"}
 	for n := 0; n < b.N; n++ {
-		err := modifyMessageTengo("bench.tengo", msg)
+		err := modifyInMessageTengo("bench.tengo", msg)
 		if err != nil {
 			return
 		}

gateway/router.go

@@ -110,7 +110,9 @@ func (r *Router) disableBridge(br *bridge.Bridge, err error) bool {
 	if r.BridgeValues().General.IgnoreFailureOnStart {
 		r.logger.Error(err)
 		// setting this bridge empty
-		*br = bridge.Bridge{}
+		*br = bridge.Bridge{
+			Log: br.Log,
+		}
 		return true
 	}
 	return false
@@ -160,7 +162,7 @@ func (r *Router) handleReceive() {
 				// For some bridges we always add/update the message ID.
 				// This is necessary as msgIDs will change if a bridge returns
 				// a different ID in response to edits.
-				if !exists || msg.Protocol == "discord" {
+				if !exists {
 					gw.Messages.Add(msg.Protocol+" "+msg.ID, msgIDs)
 				}
 			}

go.mod

@@ -3,61 +3,134 @@ module github.com/42wim/matterbridge
 require (
 	github.com/42wim/go-gitter v0.0.0-20170828205020-017310c2d557
 	github.com/Baozisoftware/qrcode-terminal-go v0.0.0-20170407111555-c0650d8dff0f
-	github.com/Jeffail/gabs v1.1.1 // indirect
+	github.com/Benau/tgsconverter v0.0.0-20210809170556-99f4a4f6337f
-	github.com/Philipp15b/go-steam v1.0.1-0.20190816133340-b04c5a83c1c0
+	github.com/Philipp15b/go-steam v1.0.1-0.20200727090957-6ae9b3c0a560
-	github.com/Rhymen/go-whatsapp v0.1.1-0.20200421062035-31e8111ac334
+	github.com/Rhymen/go-whatsapp v0.1.2-0.20211102134409-31a2e740845c
-	github.com/d5/tengo/v2 v2.6.0
+	github.com/SevereCloud/vksdk/v2 v2.10.0
+	github.com/d5/tengo/v2 v2.8.0
 	github.com/davecgh/go-spew v1.1.1
-	github.com/fsnotify/fsnotify v1.4.9
+	github.com/fsnotify/fsnotify v1.5.1
 	github.com/go-telegram-bot-api/telegram-bot-api v1.0.1-0.20200524105306-7434b0456e81
-	github.com/gomarkdown/markdown v0.0.0-20200609195525-3f9352745725
+	github.com/gomarkdown/markdown v0.0.0-20210918233619-6c1113f12c4a
-	github.com/google/gops v0.3.10
+	github.com/google/gops v0.3.22
-	github.com/gopackage/ddp v0.0.0-20170117053602-652027933df4 // indirect
+	github.com/gorilla/schema v1.2.0
-	github.com/gorilla/schema v1.1.0
 	github.com/gorilla/websocket v1.4.2
 	github.com/hashicorp/golang-lru v0.5.4
-	github.com/hpcloud/tail v1.0.0 // indirect
 	github.com/jpillora/backoff v1.0.0
-	github.com/keybase/go-keybase-chat-bot v0.0.0-20200505163032-5cacf52379da
+	github.com/keybase/go-keybase-chat-bot v0.0.0-20211004153716-fd2ee4d6be11
-	github.com/labstack/echo/v4 v4.1.16
+	github.com/kyokomi/emoji/v2 v2.2.8
-	github.com/lrstanley/girc v0.0.0-20190801035559-4fc93959e1a7
+	github.com/labstack/echo/v4 v4.6.1
-	github.com/matterbridge/Rocket.Chat.Go.SDK v0.0.0-20200411204219-d5c18ce75048
+	github.com/lrstanley/girc v0.0.0-20210611213246-771323f1624b
-	github.com/matterbridge/discordgo v0.21.2-0.20200718144317-01fe5db6c78d
+	github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16
-	github.com/matterbridge/emoji v2.1.1-0.20191117213217-af507f6b02db+incompatible
+	github.com/matterbridge/Rocket.Chat.Go.SDK v0.0.0-20211016222428-79310a412696
-	github.com/matterbridge/go-xmpp v0.0.0-20200418225040-c8a3a57b4050
+	github.com/matterbridge/discordgo v0.21.2-0.20210201201054-fb39a175b4f7
-	github.com/matterbridge/gomatrix v0.0.0-20200209224845-c2104d7936a6
+	github.com/matterbridge/go-xmpp v0.0.0-20210731150933-5702291c239f
-	github.com/matterbridge/gozulipbot v0.0.0-20190212232658-7aa251978a18
+	github.com/matterbridge/gozulipbot v0.0.0-20211023205727-a19d6c1f3b75
 	github.com/matterbridge/logrus-prefixed-formatter v0.5.3-0.20200523233437-d971309a77ba
-	github.com/mattermost/mattermost-server v5.5.0+incompatible
+	github.com/matterbridge/matterclient v0.0.0-20211024214211-22e762684b4a
-	github.com/mattn/godown v0.0.0-20200217152941-afc959f6a561
+	github.com/mattermost/mattermost-server/v5 v5.39.0
-	github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect
+	github.com/mattermost/mattermost-server/v6 v6.0.2
-	github.com/missdeer/golib v1.0.3
+	github.com/mattn/godown v0.0.1
-	github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474 // indirect
+	github.com/missdeer/golib v1.0.4
-	github.com/mrexodia/wray v0.0.0-20160318003008-78a2c1f284ff // indirect
 	github.com/nelsonken/gomf v0.0.0-20180504123937-a9dd2f9deae9
-	github.com/nicksnyder/go-i18n v1.4.0 // indirect
-	github.com/onsi/ginkgo v1.6.0 // indirect
-	github.com/onsi/gomega v1.4.1 // indirect
 	github.com/paulrosania/go-charset v0.0.0-20190326053356-55c9d7a5834c
-	github.com/pborman/uuid v0.0.0-20160216163710-c55201b03606 // indirect
+	github.com/rs/xid v1.3.0
-	github.com/rs/xid v1.2.1
+	github.com/russross/blackfriday v1.6.0
-	github.com/russross/blackfriday v1.5.2
 	github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca
-	github.com/shazow/ssh-chat v1.8.3-0.20200308224626-80ddf1f43a98
+	github.com/shazow/ssh-chat v1.10.1
-	github.com/sirupsen/logrus v1.6.0
+	github.com/sirupsen/logrus v1.8.1
-	github.com/slack-go/slack v0.6.5
+	github.com/slack-go/slack v0.9.5
-	github.com/spf13/viper v1.7.0
+	github.com/spf13/viper v1.9.0
-	github.com/stretchr/testify v1.5.1
+	github.com/stretchr/testify v1.7.0
+	github.com/vincent-petithory/dataurl v0.0.0-20191104211930-d1553a71de50
 	github.com/writeas/go-strip-markdown v2.0.1+incompatible
-	github.com/x-cray/logrus-prefixed-formatter v0.5.2 // indirect
+	github.com/yaegashi/msgraph.go v0.1.4
-	github.com/yaegashi/msgraph.go v0.1.3
+	github.com/zfjagann/golang-ring v0.0.0-20210116075443-7c86fdb43134
-	github.com/zfjagann/golang-ring v0.0.0-20190304061218-d34796e0a6c2
+	golang.org/x/image v0.0.0-20211028202545-6944b10bf410
-	golang.org/x/image v0.0.0-20200618115811-c13761719519
+	golang.org/x/oauth2 v0.0.0-20211028175245-ba495a64dcb5
-	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
+	gomod.garykim.dev/nc-talk v0.3.0
-	gomod.garykim.dev/nc-talk v0.0.1
+	gopkg.in/olahol/melody.v1 v1.0.0-20170518105555-d52139073376
-	gopkg.in/fsnotify.v1 v1.4.7 // indirect
+	layeh.com/gumble v0.0.0-20200818122324-146f9205029b
-	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
-	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 )
 
-go 1.13
+require (
+	github.com/Benau/go_rlottie v0.0.0-20210807002906-98c1b2421989 // indirect
+	github.com/Jeffail/gabs v1.4.0 // indirect
+	github.com/apex/log v1.9.0 // indirect
+	github.com/av-elier/go-decimal-to-rational v0.0.0-20191127152832-89e6aad02ecf // indirect
+	github.com/blang/semver v3.5.1+incompatible // indirect
+	github.com/dustin/go-humanize v1.0.0 // indirect
+	github.com/dyatlov/go-opengraph v0.0.0-20210112100619-dae8665a5b09 // indirect
+	github.com/francoispqt/gojay v1.2.13 // indirect
+	github.com/go-asn1-ber/asn1-ber v1.5.3 // indirect
+	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/uuid v1.2.0 // indirect
+	github.com/gopackage/ddp v0.0.3 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.11 // indirect
+	github.com/kettek/apng v0.0.0-20191108220231-414630eed80f // indirect
+	github.com/klauspost/cpuid/v2 v2.0.6 // indirect
+	github.com/labstack/gommon v0.3.0 // indirect
+	github.com/magiconair/properties v1.8.5 // indirect
+	github.com/mattermost/go-i18n v1.11.1-0.20211013152124-5c415071e404 // indirect
+	github.com/mattermost/ldap v0.0.0-20201202150706-ee0e6284187d // indirect
+	github.com/mattermost/logr v1.0.13 // indirect
+	github.com/mattermost/logr/v2 v2.0.10 // indirect
+	github.com/mattn/go-colorable v0.1.8 // indirect
+	github.com/mattn/go-isatty v0.0.14 // indirect
+	github.com/mattn/go-runewidth v0.0.13 // indirect
+	github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
+	github.com/minio/md5-simd v1.1.2 // indirect
+	github.com/minio/minio-go/v7 v7.0.11 // indirect
+	github.com/minio/sha256-simd v1.0.0 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/mitchellh/mapstructure v1.4.2 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.1 // indirect
+	github.com/monaco-io/request v1.0.5 // indirect
+	github.com/mreiferson/go-httpclient v0.0.0-20201222173833-5e475fde3a4d // indirect
+	github.com/mrexodia/wray v0.0.0-20160318003008-78a2c1f284ff // indirect
+	github.com/pborman/uuid v1.2.1 // indirect
+	github.com/pelletier/go-toml v1.9.4 // indirect
+	github.com/philhofer/fwd v1.1.1 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/rickb777/date v1.12.4 // indirect
+	github.com/rickb777/plural v1.2.0 // indirect
+	github.com/rivo/uniseg v0.2.0 // indirect
+	github.com/shazow/rateio v0.0.0-20200113175441-4461efc8bdc4 // indirect
+	github.com/sizeofint/webpanimation v0.0.0-20210809145948-1d2b32119882 // indirect
+	github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect
+	github.com/spf13/afero v1.6.0 // indirect
+	github.com/spf13/cast v1.4.1 // indirect
+	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/subosito/gotenv v1.2.0 // indirect
+	github.com/technoweenie/multipartstreamer v1.0.1 // indirect
+	github.com/tinylib/msgp v1.1.6 // indirect
+	github.com/valyala/bytebufferpool v1.0.0 // indirect
+	github.com/valyala/fasttemplate v1.2.1 // indirect
+	github.com/wiggin77/cfg v1.0.2 // indirect
+	github.com/wiggin77/merror v1.0.3 // indirect
+	github.com/wiggin77/srslog v1.0.1 // indirect
+	go.uber.org/atomic v1.8.0 // indirect
+	go.uber.org/multierr v1.7.0 // indirect
+	go.uber.org/zap v1.17.0 // indirect
+	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect
+	golang.org/x/net v0.0.0-20210913180222-943fd674d43e // indirect
+	golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0 // indirect
+	golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect
+	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/protobuf v1.27.1 // indirect
+	gopkg.in/ini.v1 v1.63.2 // indirect
+	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+)
+
+go 1.17

matterbridge.go

@@ -10,15 +10,13 @@ import (
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/42wim/matterbridge/gateway"
 	"github.com/42wim/matterbridge/gateway/bridgemap"
+	"github.com/42wim/matterbridge/version"
 	"github.com/google/gops/agent"
 	prefixed "github.com/matterbridge/logrus-prefixed-formatter"
 	"github.com/sirupsen/logrus"
 )
 
 var (
-	version = "1.18.0"
-	githash string
-
 	flagConfig  = flag.String("conf", "matterbridge.toml", "config file")
 	flagDebug   = flag.Bool("debug", false, "enable debug")
 	flagVersion = flag.Bool("version", false, "show version")
@@ -28,7 +26,7 @@ var (
 func main() {
 	flag.Parse()
 	if *flagVersion {
-		fmt.Printf("version: %s %s\n", version, githash)
+		fmt.Printf("version: %s %s\n", version.Release, version.GitHash)
 		return
 	}
 
@@ -43,8 +41,8 @@ func main() {
 		}
 	}
 
-	logger.Printf("Running version %s %s", version, githash)
+	logger.Printf("Running version %s %s", version.Release, version.GitHash)
-	if strings.Contains(version, "-dev") {
+	if strings.Contains(version.Release, "-dev") {
 		logger.Println("WARNING: THIS IS A DEVELOPMENT VERSION. Things may break.")
 	}
 

matterbridge.toml.sample

@@ -9,12 +9,12 @@
 [irc]
 
 #You can configure multiple servers "[irc.name]" or "[irc.name2]"
-#In this example we use [irc.freenode]
+#In this example we use [irc.libera]
 #REQUIRED
-[irc.freenode]
+[irc.libera]
 #irc server to connect to.
 #REQUIRED
-Server="irc.freenode.net:6667"
+Server="irc.libera.chat:6667"
 
 #Password for irc server (if necessary)
 #OPTIONAL (default "")
@@ -24,7 +24,7 @@ Password=""
 #OPTIONAL (default false)
 UseTLS=false
 
-#Enable SASL (PLAIN) authentication. (freenode requires this from eg AWS hosts)
+#Enable SASL (PLAIN) authentication. (libera requires this from eg AWS hosts)
 #It uses NickServNick and NickServPassword as login and password
 #OPTIONAL (default false)
 UseSASL=false
@@ -55,7 +55,15 @@ Charset=""
 #REQUIRED
 Nick="matterbot"
 
-#If you registered your bot with a service like Nickserv on freenode.
+#Real name/gecos displayed in e.g. /WHOIS and /WHO
+#OPTIONAL (defaults to the nick)
+RealName="Matterbridge instance on IRC"
+
+#IRC username/ident preceding the hostname in hostmasks and /WHOIS
+#OPTIONAL (defaults to the nick)
+UserName="bridge"
+
+#If you registered your bot with a service like Nickserv on libera.
 #Also being used when UseSASL=true
 #
 #Note: if you want do to quakenet auth, set NickServNick="Q@CServe.quakenet.org"
@@ -76,20 +84,24 @@ MessageDelay=1300
 
 #Maximum amount of messages to hold in queue. If queue is full
 #messages will be dropped.
-#<message clipped> will be add to the message that fills the queue.
+#<clipped message> will be add to the message that fills the queue.
 #OPTIONAL (default 30)
 MessageQueue=30
 
 #Maximum length of message sent to irc server. If it exceeds
-#<message clipped> will be add to the message.
+#<clipped message> will be add to the message.
 #OPTIONAL (default 400)
 MessageLength=400
 
-#Split messages on MessageLength instead of showing the <message clipped>
+#Split messages on MessageLength instead of showing the <clipped message>
 #WARNING: this could lead to flooding
 #OPTIONAL (default false)
 MessageSplit=false
 
+#Message to show when a message is too big
+#Default "<clipped message>"
+MessageClipped="<clipped message>"
+
 #Delay in seconds to rejoin a channel when kicked
 #OPTIONAL (default 0)
 RejoinDelay=0
@@ -103,6 +115,12 @@ ColorNicks=false
 #OPTIONAL (default empty)
 RunCommands=["PRIVMSG user hello","PRIVMSG chanserv something"]
 
+#PingDelay specifies how long to wait to send a ping to the irc server.
+#You can use s for second, m for minute
+#String
+#OPTIONAL (default 1m)
+PingDelay="1m"
+
 #StripMarkdown strips markdown from messages
 #OPTIONAL (default false)
 StripMarkdown=false
@@ -187,6 +205,19 @@ ShowTopicChange=false
 #OPTIONAL (default 0)
 JoinDelay=0
 
+#Use the optional RELAYMSG extension for username spoofing on IRC.
+#This requires an IRCd that supports the draft/relaymsg specification: currently this includes
+#Oragono 2.4.0+ and InspIRCd 3 with the m_relaymsg contrib module.
+#See https://github.com/42wim/matterbridge/issues/667#issuecomment-634214165 for more details.
+#Spoofed nicks will use the configured RemoteNickFormat, replacing reserved IRC characters
+#(!+%@&#$:'"?*,.) with a hyphen (-).
+#On most configurations, the RemoteNickFormat must include a separator character such as "/".
+#You should make sure that the settings here match your IRCd.
+#This option overrides ColorNicks.
+#OPTIONAL (default false)
+UseRelayMsg=false
+#RemoteNickFormat="{NICK}/{PROTOCOL}"
+
 ###################################################################
 #XMPP section
 ###################################################################
@@ -200,12 +231,16 @@ JoinDelay=0
 #REQUIRED
 Server="jabber.example.com:5222"
 
+#Use anonymous MUC login
+#OPTIONAL (default false)
+Anonymous=false
+
 #Jid
-#REQUIRED
+#REQUIRED if Anonymous=false
 Jid="user@example.com"
 
 #Password
-#REQUIRED
+#REQUIRED if Anonymous=false
 Password="yourpass"
 
 #MUC
@@ -221,6 +256,10 @@ Nick="xmppbot"
 #OPTIONAL (default false)
 SkipTLSVerify=true
 
+#Enable to use plaintext connection to your XMPP server.
+#OPTIONAL (default false)
+NoTLS=true
+
 ## RELOADABLE SETTINGS
 ## Settings below can be reloaded by editing the file
 
@@ -287,6 +326,11 @@ StripNick=false
 #OPTIONAL (default false)
 ShowTopicChange=false
 
+#Enable sending messages using a webhook instead of regular MUC messages.
+#Only works with a prosody server using mod_slack_webhook. Does not support editing.
+#OPTIONAL (default "")
+WebhookURL="https://yourdomain/prosody/msg/someid"
+
 ###################################################################
 #mattermost section
 ###################################################################
@@ -798,6 +842,10 @@ PreserveThreading=false
 #OPTIONAL (default false)
 ShowUserTyping=false
 
+#Message to show when a message is too big
+#Default "<clipped message>"
+MessageClipped="<clipped message>"
+
 ###################################################################
 #discord section
 ###################################################################
@@ -820,6 +868,14 @@ Server="yourservername"
 ## All settings below can be reloaded by editing the file.
 ## They are also all optional.
 
+# AllowMention controls which mentions are allowed. If not specified, all mentions are allowed.
+# Note that even when a mention is not allowed, it will still be displayed nicely and be clickable. It just prevents the ping/notification.
+#
+# "everyone" allows @everyone and @here mentions
+# "roles" allows @role mentions
+# "users" allows @user mentions
+AllowMention=["everyone", "roles", "users"]
+
 # ShowEmbeds shows the title, description and URL of embedded messages (sent by other bots)
 ShowEmbeds=false
 
@@ -836,10 +892,11 @@ UseUserName=false
 # UseDiscriminator appends the `#xxxx` discriminator when used with UseUserName
 UseDiscriminator=false
 
-# WebhookURL sends messages in the style of puppets.
+# AutoWebhooks automatically configures message sending in the style of puppets.
-# This only works if you have one discord channel, if you have multiple discord channels you'll have to specify it in the gateway config
+# This is an easier alternative to manually configuring "WebhookURL" for each gateway,
-# Example: "https://discordapp.com/api/webhooks/1234/abcd_xyzw"
+# as turning this on will automatically load or create webhooks for each channel.
-WebhookURL=""
+# This feature requires the "Manage Webhooks" permission (either globally or as per-channel).
+AutoWebhooks=false
 
 # EditDisable disables sending of edits to other bridges
 EditDisable=false
@@ -924,6 +981,10 @@ ShowTopicChange=false
 # Supported from the following bridges: slack
 SyncTopic=false
 
+#Message to show when a message is too big
+#Default "<clipped message>"
+MessageClipped="<clipped message>"
+
 ###################################################################
 #telegram section
 ###################################################################
@@ -982,6 +1043,13 @@ QuoteFormat="{MESSAGE} (re @{QUOTENICK}: {QUOTEMESSAGE})"
 #OPTIONAL (default false)
 MediaConvertWebPToPNG=false
 
+#Convert Tgs (Telegram animated sticker) images to PNG before upload.
+#This is useful when your bridge also contains platforms that do not support animated WebP files, like Discord.
+#This requires the external dependency `lottie`, which can be installed like this:
+#`pip install lottie cairosvg`
+#https://github.com/42wim/matterbridge/issues/874
+#MediaConvertTgs="png"
+
 #Disable sending of edits to other bridges
 #OPTIONAL (default false)
 EditDisable=false
@@ -1201,12 +1269,16 @@ ShowTopicChange=false
 #REQUIRED
 Server="https://matrix.org"
 
-#login/pass of your bot.
+#Authentication for your bot.
+#You can use either login/password OR mxid/token. The latter will be preferred if found.
 #Use a dedicated user for this and not your own!
 #Messages sent from this user will not be relayed to avoid loops.
 #REQUIRED
 Login="yourlogin"
 Password="yourpass"
+#OR
+MxID="@yourlogin:domain.tld"
+Token="tokenforthebotuser"
 
 #Whether to send the homeserver suffix. eg ":matrix.org" in @username:matrix.org
 #to other bridges, or only send "username".(true only sends username)
@@ -1221,12 +1293,8 @@ HTMLDisable=false
 ## RELOADABLE SETTINGS
 ## Settings below can be reloaded by editing the file
 
-#Whether to prefix messages from other bridges to matrix with the sender's nick.
+# UseUserName shows the username instead of the server nickname
-#Useful if username overrides for incoming webhooks isn't enabled on the
+UseUserName=false
-#matrix server. If you set PrefixMessagesWithNick to true, each message
-#from bridge to matrix will by default be prefixed by the RemoteNickFormat setting. i
-#OPTIONAL (default false)
-PrefixMessagesWithNick=false
 
 #Nicks you want to ignore.
 #Regular expressions supported
@@ -1384,9 +1452,7 @@ StripNick=false
 ShowTopicChange=false
 
 ###################################################################
-#
 # NCTalk (Nextcloud Talk)
-#
 ###################################################################
 
 [nctalk.bridge]
@@ -1394,16 +1460,86 @@ ShowTopicChange=false
 # Url of your Nextcloud server
 Server = "https://cloud.youdomain.me"
 
+# Enable to not verify the certificate on your Nextcloud server.
+# e.g. when using selfsigned certificates
+# OPTIONAL (default false)
+SkipTLSVerify=true
+
 # Username of the bot
 Login = "talkuser"
 
 # Password of the bot
 Password = "talkuserpass"
 
+# Suffix for Guest Users
+GuestSuffix = " (Guest)"
+
+# Separate display name (Note: needs to be configured from Nextcloud Talk to work)
+SeparateDisplayName=false
+
+###################################################################
+# Mumble
+###################################################################
+
+[mumble.bridge]
+
+# Host and port of your Mumble server
+Server = "mumble.yourdomain.me:64738"
+
+# Nickname to log in as
+Nick = "matterbridge"
+
+# Some servers require a password
+# OPTIONAL (default empty)
+Password = "serverpasswordhere"
+
+# User comment to set on the Mumble user, visible to other users.
+# OPTIONAL (default empty)
+UserComment="I am bridging text messages between this channel and #general on irc.yourdomain.me"
+
+# Self-signed TLS client certificate + private key used to connect to
+# Mumble.  This is required if you want to register the matterbridge
+# user on your Mumble server, so its nick becomes reserved.
+# You can generate a keypair using e.g.
+#
+#     openssl req -x509 -newkey rsa:2048 -nodes -days 10000 \
+#             -keyout mumble.key -out mumble.crt
+#
+# To actually register the matterbridege user, connect to Mumble as an
+# admin, right click on the user and click "Register".
+#
+# OPTIONAL (default empty)
+TLSClientCertificate="mumble.crt"
+TLSClientKey="mumble.key"
+
+# TLS CA certificate used to validate the Mumble server.
+# OPTIONAL (defaults to Go system CA)
+TLSCACertificate=mumble-ca.crt
+
+# Enable to not verify the certificate on your Mumble server.
+# e.g. when using selfsigned certificates
+# OPTIONAL (default false)
+SkipTLSVerify=false
+
+#Message to show when a message is too big
+#Default "<clipped message>"
+MessageClipped="<clipped message>"
+
+###################################################################
+#VK
 ###################################################################
 #
+[vk.myvk]
+#Group access token
+#See https://vk.com/dev/bots_docs
+Token="Yourtokenhere"
+
+#Group ID
+#For example in URL https://vk.com/public168963511 group ID is 168963511
+GroupID=123456789
+
+###################################################################
 # WhatsApp
-#
 ###################################################################
 
 [whatsapp.bridge]
@@ -1430,9 +1566,7 @@ Label="Organization"
 
 
 ###################################################################
-#
 # zulip
-#
 ###################################################################
 
 [zulip]
@@ -1540,6 +1674,8 @@ Buffer=1000
 
 #Bearer token used for authentication
 #curl -H "Authorization: Bearer token" http://localhost:4242/api/messages
+# https://github.com/vi/websocat
+# websocat -H="Authorization: Bearer token" ws://127.0.0.1:4242/api/websocket
 #OPTIONAL (no authorization if token is empty)
 Token="mytoken"
 
@@ -1563,7 +1699,8 @@ RemoteNickFormat="{NICK}"
 ## Settings below can be reloaded by editing the file
 
 #RemoteNickFormat defines how remote users appear on this bridge
-#The string "{NICK}" (case sensitive) will be replaced by the actual nick / username.
+#The string "{NICK}" (case sensitive) will be replaced by the actual nick.
+#The string "{USERID}" (case sensitive) will be replaced by the user ID.
 #The string "{BRIDGE}" (case sensitive) will be replaced by the sending bridge
 #The string "{LABEL}" (case sensitive) will be replaced by label= field of the sending bridge
 #The string "{PROTOCOL}" (case sensitive) will be replaced by the protocol used by the bridge
@@ -1589,7 +1726,7 @@ StripNick=false
 #The MediaServerDownload will be used so that bridges without native uploading support:
 #gitter, irc and xmpp will be shown links to the files on MediaServerDownload
 #
-#More information https://github.com/42wim/matterbridge/wiki/Mediaserver-setup-%5Badvanced%5D
+#More information https://github.com/42wim/matterbridge/wiki/Mediaserver-setup-%28advanced%29
 #OPTIONAL (default empty)
 MediaServerUpload="https://user:pass@yourserver.com/upload"
 #OPTIONAL (default empty)
@@ -1625,7 +1762,7 @@ IgnoreFailureOnStart=false
 #writing, or if the value is empty. Note that the log won't roll, so
 #you might want to use logrotate(8) with this feature.
 #OPTIONAL (default empty)
-LogFile=/var/log/matterbridge.log
+LogFile="/var/log/matterbridge.log"
 
 ###################################################################
 #Tengo configuration
@@ -1638,7 +1775,7 @@ LogFile=/var/log/matterbridge.log
 #This script will receive every incoming message and can be used to modify the Username and the Text of that message.
 #The script will have the following global variables:
 #to modify: msgUsername and msgText
-#to read: msgChannel and msgAccount
+#to read: msgUserID, msgChannel, msgAccount
 #
 #The script is reloaded on every message, so you can modify the script on the fly.
 #
@@ -1662,9 +1799,12 @@ InMessage="example.tengo"
 #read-only:
 #inAccount, inProtocol, inChannel, inGateway, inEvent
 #outAccount, outProtocol, outChannel, outGateway, outEvent
+#msgUserID
 #
 #read-write:
-#msgText, msgUsername
+#msgText, msgUsername, msgDrop
+#
+#msgDrop is a bool which is default false, when set true this message will be dropped
 #
 #The script is reloaded on every message, so you can modify the script on the fly.
 #
@@ -1677,7 +1817,7 @@ OutMessage="example.tengo"
 #RemoteNickFormat allows you to specify the location of a tengo (https://github.com/d5/tengo/) script.
 #The script will have the following global variables:
 #to modify: result
-#to read: channel, bridge, gateway, protocol, nick
+#to read: channel, bridge, gateway, protocol, nick, msgUserID
 #
 #The result will be set in {TENGO} in the RemoteNickFormat key of every bridge where {TENGO} is specified
 #
@@ -1715,7 +1855,7 @@ enable=true
 
     # account specified above
     # REQUIRED
-    account="irc.freenode"
+    account="irc.libera"
 
     # The channel key in each gateway is mapped to a similar group chat ID on the chat platform
     # To find the group chat ID for different platforms, refer to the table below
@@ -1738,10 +1878,12 @@ enable=true
     # -------------------------------------------------------------------------------------------------------------------------------------
     #   msteams  |      threadId      |    19:82abcxx@thread.skype    | You'll find the threadId in the URL
     # -------------------------------------------------------------------------------------------------------------------------------------
+    #   mumble   |    channel id      |              42               | The channel ID, as shown in the channel's "Edit" window
+    # -------------------------------------------------------------------------------------------------------------------------------------
     # rocketchat |      channel       |            #channel           | # is required for private channels too
     # -------------------------------------------------------------------------------------------------------------------------------------
     #   slack    |   channel name     |            general            | Do not include the # symbol
-    #            |    channel id      |           ID:C123456          | The underlying ID of a channel. This doesn't work with
+    #            |    channel id      |           ID:C123456          | The underlying ID of a channel. This doesn't work with webhooks.
     # -------------------------------------------------------------------------------------------------------------------------------------
     #   steam    |      chatid        |         example needed        | The number in the URL when you click "enter chat room" in the browser
     # -------------------------------------------------------------------------------------------------------------------------------------
@@ -1749,6 +1891,8 @@ enable=true
     # -------------------------------------------------------------------------------------------------------------------------------------
     #  telegram  |      chatid        |          -123456789           | A large negative number. see https://www.linkedin.com/pulse/telegram-bots-beginners-marco-frau
     # -------------------------------------------------------------------------------------------------------------------------------------
+    #  vk        |      peerid        |          2000000002           | A number that starts form 2000000000. Use --debug and send any message in chat to get PeerID in the logs
+    # -------------------------------------------------------------------------------------------------------------------------------------
     #  whatsapp  |     group JID      | 48111222333-123455678999@g.us | A unique group JID. If you specify an empty string, bridge will list all the possibilities
     #            |    "Group Name"    |         "Family Chat"         | if you specify a group name, the bridge will find hint the JID to specify. Names can change over time and are not stable.
     # -------------------------------------------------------------------------------------------------------------------------------------
@@ -1769,7 +1913,7 @@ enable=true
 
     #[[gateway.out]] specifies the account and channels we will sent messages to.
     [[gateway.out]]
-    account="irc.freenode"
+    account="irc.libera"
     channel="#testing"
 
         #OPTIONAL - only used for IRC and XMPP protocols at the moment
@@ -1788,13 +1932,16 @@ enable=true
         #OPTIONAL - your irc / xmpp channel key
         key="yourkey"
 
+    # Discord specific gateway options
     [[gateway.inout]]
     account="discord.game"
     channel="mygreatgame"
 
-        #OPTIONAL - webhookurl only works for discord (it needs a different URL for each cahnnel)
         [gateway.inout.options]
-        webhookurl="https://discordapp.com/api/webhooks/123456789123456789/C9WPqExYWONPDZabcdef-def1434FGFjstasJX9pYht73y"
+        # WebhookURL sends messages in the style of "puppets". You must configure a webhook URL for each channel you want to bridge.
+        # If you have more than one channel and don't wnat to configure each channel manually, see the "AutoWebhooks" option in the gateway config.
+        # Example: "https://discord.com/api/webhooks/1234/abcd_xyzw"
+        WebhookURL=""
 
     [[gateway.inout]]
     account="zulip.streamchat"

matterclient/channels.go

@@ -4,7 +4,7 @@ import (
 	"errors"
 	"strings"
 
-	"github.com/mattermost/mattermost-server/model"
+	"github.com/mattermost/mattermost-server/v5/model"
 )
 
 // GetChannels returns all channels we're members off
@@ -167,7 +167,7 @@ func (m *MMClient) JoinChannel(channelId string) error { //nolint:golint
 }
 
 func (m *MMClient) UpdateChannelsTeam(teamID string) error {
-	mmchannels, resp := m.Client.GetChannelsForTeamForUser(teamID, m.User.Id, "")
+	mmchannels, resp := m.Client.GetChannelsForTeamForUser(teamID, m.User.Id, false, "")
 	if resp.Error != nil {
 		return errors.New(resp.Error.DetailedError)
 	}

matterclient/helpers.go

@@ -13,7 +13,7 @@ import (
 
 	"github.com/gorilla/websocket"
 	"github.com/jpillora/backoff"
-	"github.com/mattermost/mattermost-server/model"
+	"github.com/mattermost/mattermost-server/v5/model"
 )
 
 func (m *MMClient) doLogin(firstConnection bool, b *backoff.Backoff) error {
@@ -154,7 +154,7 @@ func (m *MMClient) initUser() error {
 
 		t := &Team{Team: team, Users: usermap, Id: team.Id}
 
-		mmchannels, resp := m.Client.GetChannelsForTeamForUser(team.Id, m.User.Id, "")
+		mmchannels, resp := m.Client.GetChannelsForTeamForUser(team.Id, m.User.Id, false, "")
 		if resp.Error != nil {
 			return resp.Error
 		}

matterclient/matterclient.go

@@ -11,7 +11,7 @@ import (
 	lru "github.com/hashicorp/golang-lru"
 	"github.com/jpillora/backoff"
 	prefixed "github.com/matterbridge/logrus-prefixed-formatter"
-	"github.com/mattermost/mattermost-server/model"
+	"github.com/mattermost/mattermost-server/v5/model"
 	"github.com/sirupsen/logrus"
 )
 
@@ -69,6 +69,7 @@ type MMClient struct {
 	logger     *logrus.Entry
 	rootLogger *logrus.Logger
 	lruCache   *lru.Cache
+	allevents  bool
 }
 
 // New will instantiate a new Matterclient with the specified login details without connecting.
@@ -119,6 +120,10 @@ func (m *MMClient) SetLogLevel(level string) {
 	}
 }
 
+func (m *MMClient) EnableAllEvents() {
+	m.allevents = true
+}
+
 // Login tries to connect the client with the loging details with which it was initialized.
 func (m *MMClient) Login() error {
 	// check if this is a first connect or a reconnection
@@ -220,6 +225,10 @@ func (m *MMClient) WsReceiver() {
 					continue
 				}
 			}
+			if m.allevents {
+				m.MessageChan <- msg
+				continue
+			}
 			switch msg.Raw.Event {
 			case model.WEBSOCKET_EVENT_USER_ADDED,
 				model.WEBSOCKET_EVENT_USER_REMOVED,

matterclient/messages.go

@@ -3,13 +3,13 @@ package matterclient
 import (
 	"strings"
 
-	"github.com/mattermost/mattermost-server/model"
+	"github.com/mattermost/mattermost-server/v5/model"
 )
 
 func (m *MMClient) parseActionPost(rmsg *Message) {
 	// add post to cache, if it already exists don't relay this again.
 	// this should fix reposts
-	if ok, _ := m.lruCache.ContainsOrAdd(digestString(rmsg.Raw.Data["post"].(string)), true); ok {
+	if ok, _ := m.lruCache.ContainsOrAdd(digestString(rmsg.Raw.Data["post"].(string)), true); ok && rmsg.Raw.Event != model.WEBSOCKET_EVENT_POST_DELETED {
 		m.logger.Debugf("message %#v in cache, not processing again", rmsg.Raw.Data["post"].(string))
 		rmsg.Text = ""
 		return
@@ -111,7 +111,7 @@ func (m *MMClient) GetFileLinks(filenames []string) []string {
 }
 
 func (m *MMClient) GetPosts(channelId string, limit int) *model.PostList { //nolint:golint
-	res, resp := m.Client.GetPostsForChannel(channelId, 0, limit, "")
+	res, resp := m.Client.GetPostsForChannel(channelId, 0, limit, "", true)
 	if resp.Error != nil {
 		return nil
 	}
@@ -119,7 +119,7 @@ func (m *MMClient) GetPosts(channelId string, limit int) *model.PostList { //nol
 }
 
 func (m *MMClient) GetPostsSince(channelId string, time int64) *model.PostList { //nolint:golint
-	res, resp := m.Client.GetPostsSince(channelId, time)
+	res, resp := m.Client.GetPostsSince(channelId, time, true)
 	if resp.Error != nil {
 		return nil
 	}

matterclient/users.go

@@ -4,7 +4,7 @@ import (
 	"errors"
 	"time"
 
-	"github.com/mattermost/mattermost-server/model"
+	"github.com/mattermost/mattermost-server/v5/model"
 )
 
 func (m *MMClient) GetNickName(userId string) string { //nolint:golint

tgs.Dockerfile

@@ -0,0 +1,35 @@
+FROM alpine AS builder
+
+COPY . /go/src/matterbridge
+RUN apk add \
+    go \
+    git \
+  && cd /go/src/matterbridge \
+  && go build -mod vendor -ldflags "-X github.com/42wim/matterbridge/version.GitHash=$(git log --pretty=format:'%h' -n 1)" -o /bin/matterbridge
+
+FROM alpine
+RUN apk --no-cache add \
+    ca-certificates \
+    cairo \
+    libjpeg-turbo \
+    libwebp-dev \
+    mailcap \
+    py3-webencodings \
+    python3 \
+  && apk --no-cache add --virtual .compile \
+    gcc \
+    libffi-dev \
+    libjpeg-turbo-dev \
+    musl-dev \
+    py3-pip \
+    py3-wheel \
+    python3-dev \
+    zlib-dev \
+  && pip3 install --no-cache-dir lottie[PNG] \
+  && apk --no-cache del .compile
+
+COPY --from=builder /bin/matterbridge /bin/matterbridge
+RUN mkdir /etc/matterbridge \
+  && touch /etc/matterbridge/matterbridge.toml \
+  && ln -sf /matterbridge.toml /etc/matterbridge/matterbridge.toml
+ENTRYPOINT ["/bin/matterbridge", "-conf", "/etc/matterbridge/matterbridge.toml"]

vendor/github.com/Benau/go_rlottie/LICENSE

@@ -0,0 +1,24 @@
+The MIT License
+
+Copyright (c) 2021, (see AUTHORS)
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation
+files (the "Software"), to deal in the Software without
+restriction, including without limitation the rights to use,
+copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following
+conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.

vendor/github.com/Benau/go_rlottie/README.md

@@ -0,0 +1 @@
+Go binding for https://github.com/Samsung/rlottie, example at https://github.com/Benau/tgsconverter

vendor/github.com/Benau/go_rlottie/binding_c_lottieanimation_capi.cpp

@@ -0,0 +1,284 @@
+/*
+ * Copyright (c) 2020 Samsung Electronics Co., Ltd. All rights reserved.
+
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "rlottie.h"
+#include "rlottie_capi.h"
+#include "vector_vdebug.h"
+
+using namespace rlottie;
+
+extern "C" {
+#include <string.h>
+#include <stdarg.h>
+
+struct Lottie_Animation_S
+{
+    std::unique_ptr<Animation>      mAnimation;
+    std::future<Surface>            mRenderTask;
+    uint32_t                       *mBufferRef;
+    LOTMarkerList                  *mMarkerList;
+};
+
+RLOTTIE_API Lottie_Animation_S *lottie_animation_from_file(const char *path)
+{
+    if (auto animation = Animation::loadFromFile(path) ) {
+        Lottie_Animation_S *handle = new Lottie_Animation_S();
+        handle->mAnimation = std::move(animation);
+        return handle;
+    } else {
+        return nullptr;
+    }
+}
+
+RLOTTIE_API Lottie_Animation_S *lottie_animation_from_data(const char *data, const char *key, const char *resourcePath)
+{
+    if (auto animation = Animation::loadFromData(data, key, resourcePath) ) {
+        Lottie_Animation_S *handle = new Lottie_Animation_S();
+        handle->mAnimation = std::move(animation);
+        return handle;
+    } else {
+        return nullptr;
+    }
+}
+
+RLOTTIE_API void lottie_animation_destroy(Lottie_Animation_S *animation)
+{
+    if (animation) {
+        if (animation->mMarkerList) {
+            for(size_t i = 0; i < animation->mMarkerList->size; i++) {
+                if (animation->mMarkerList->ptr[i].name) free(animation->mMarkerList->ptr[i].name);
+            }
+            delete[] animation->mMarkerList->ptr;
+            delete animation->mMarkerList;
+        }
+
+        if (animation->mRenderTask.valid()) {
+            animation->mRenderTask.get();
+        }
+        animation->mAnimation = nullptr;
+        delete animation;
+    }
+}
+
+RLOTTIE_API void lottie_animation_get_size(const Lottie_Animation_S *animation, size_t *width, size_t *height)
+{
+   if (!animation) return;
+
+   animation->mAnimation->size(*width, *height);
+}
+
+RLOTTIE_API double lottie_animation_get_duration(const Lottie_Animation_S *animation)
+{
+   if (!animation) return 0;
+
+   return animation->mAnimation->duration();
+}
+
+RLOTTIE_API size_t lottie_animation_get_totalframe(const Lottie_Animation_S *animation)
+{
+   if (!animation) return 0;
+
+   return animation->mAnimation->totalFrame();
+}
+
+
+RLOTTIE_API double lottie_animation_get_framerate(const Lottie_Animation_S *animation)
+{
+   if (!animation) return 0;
+
+   return animation->mAnimation->frameRate();
+}
+
+RLOTTIE_API const LOTLayerNode * lottie_animation_render_tree(Lottie_Animation_S *animation, size_t frame_num, size_t width, size_t height)
+{
+    if (!animation) return nullptr;
+
+    return animation->mAnimation->renderTree(frame_num, width, height);
+}
+
+RLOTTIE_API size_t
+lottie_animation_get_frame_at_pos(const Lottie_Animation_S *animation, float pos)
+{
+    if (!animation) return 0;
+
+    return animation->mAnimation->frameAtPos(pos);
+}
+
+RLOTTIE_API void
+lottie_animation_render(Lottie_Animation_S *animation,
+                        size_t frame_number,
+                        uint32_t *buffer,
+                        size_t width,
+                        size_t height,
+                        size_t bytes_per_line)
+{
+    if (!animation) return;
+
+    rlottie::Surface surface(buffer, width, height, bytes_per_line);
+    animation->mAnimation->renderSync(frame_number, surface);
+}
+
+RLOTTIE_API void
+lottie_animation_render_async(Lottie_Animation_S *animation,
+                              size_t frame_number,
+                              uint32_t *buffer,
+                              size_t width,
+                              size_t height,
+                              size_t bytes_per_line)
+{
+    if (!animation) return;
+
+    rlottie::Surface surface(buffer, width, height, bytes_per_line);
+    animation->mRenderTask = animation->mAnimation->render(frame_number, surface);
+    animation->mBufferRef = buffer;
+}
+
+RLOTTIE_API uint32_t *
+lottie_animation_render_flush(Lottie_Animation_S *animation)
+{
+    if (!animation) return nullptr;
+
+    if (animation->mRenderTask.valid()) {
+        animation->mRenderTask.get();
+    }
+
+    return animation->mBufferRef;
+}
+
+RLOTTIE_API void
+lottie_animation_property_override(Lottie_Animation_S *animation,
+                                   const Lottie_Animation_Property type,
+                                   const char *keypath,
+                                   ...)
+{
+    va_list prop;
+    va_start(prop, keypath);
+    const int arg_count = [type](){
+                             switch (type) {
+                              case LOTTIE_ANIMATION_PROPERTY_FILLCOLOR:
+                              case LOTTIE_ANIMATION_PROPERTY_STROKECOLOR:
+                                return 3;
+                              case LOTTIE_ANIMATION_PROPERTY_FILLOPACITY:
+                              case LOTTIE_ANIMATION_PROPERTY_STROKEOPACITY:
+                              case LOTTIE_ANIMATION_PROPERTY_STROKEWIDTH:
+                              case LOTTIE_ANIMATION_PROPERTY_TR_ROTATION:
+                                return 1;
+                              case LOTTIE_ANIMATION_PROPERTY_TR_POSITION:
+                              case LOTTIE_ANIMATION_PROPERTY_TR_SCALE:
+                                return 2;
+                              default:
+                                return 0;
+                             }
+                          }();
+    double v[3] = {0};
+    for (int i = 0; i < arg_count ; i++) {
+      v[i] = va_arg(prop, double);
+    }
+    va_end(prop);
+
+    switch(type) {
+    case LOTTIE_ANIMATION_PROPERTY_FILLCOLOR: {
+        double r = v[0];
+        double g = v[1];
+        double b = v[2];
+        if (r > 1 || r < 0 || g > 1 || g < 0 || b > 1 || b < 0) break;
+        animation->mAnimation->setValue<rlottie::Property::FillColor>(keypath, rlottie::Color(r, g, b));
+        break;
+    }
+    case LOTTIE_ANIMATION_PROPERTY_FILLOPACITY: {
+        double opacity = v[0];
+        if (opacity > 100 || opacity < 0) break;
+        animation->mAnimation->setValue<rlottie::Property::FillOpacity>(keypath, (float)opacity);
+        break;
+    }
+    case LOTTIE_ANIMATION_PROPERTY_STROKECOLOR: {
+        double r = v[0];
+        double g = v[1];
+        double b = v[2];
+        if (r > 1 || r < 0 || g > 1 || g < 0 || b > 1 || b < 0) break;
+        animation->mAnimation->setValue<rlottie::Property::StrokeColor>(keypath, rlottie::Color(r, g, b));
+        break;
+    }
+    case LOTTIE_ANIMATION_PROPERTY_STROKEOPACITY: {
+        double opacity = v[0];
+        if (opacity > 100 || opacity < 0) break;
+        animation->mAnimation->setValue<rlottie::Property::StrokeOpacity>(keypath, (float)opacity);
+        break;
+    }
+    case LOTTIE_ANIMATION_PROPERTY_STROKEWIDTH: {
+        double width = v[0];
+        if (width < 0) break;
+        animation->mAnimation->setValue<rlottie::Property::StrokeWidth>(keypath, (float)width);
+        break;
+    }
+    case LOTTIE_ANIMATION_PROPERTY_TR_POSITION: {
+        double x = v[0];
+        double y = v[1];
+        animation->mAnimation->setValue<rlottie::Property::TrPosition>(keypath, rlottie::Point((float)x, (float)y));
+        break;
+    }
+    case LOTTIE_ANIMATION_PROPERTY_TR_SCALE: {
+        double w = v[0];
+        double h = v[1];
+        animation->mAnimation->setValue<rlottie::Property::TrScale>(keypath, rlottie::Size((float)w, (float)h));
+        break;
+    }
+    case LOTTIE_ANIMATION_PROPERTY_TR_ROTATION: {
+        double r = v[0];
+        animation->mAnimation->setValue<rlottie::Property::TrRotation>(keypath, (float)r);
+        break;
+    }
+    case LOTTIE_ANIMATION_PROPERTY_TR_ANCHOR:
+    case LOTTIE_ANIMATION_PROPERTY_TR_OPACITY:
+        //@TODO handle propery update.
+        break;
+    }
+}
+
+RLOTTIE_API const LOTMarkerList*
+lottie_animation_get_markerlist(Lottie_Animation_S *animation)
+{
+   if (!animation) return nullptr;
+
+   auto markers = animation->mAnimation->markers();
+   if (markers.size() == 0) return nullptr;
+   if (animation->mMarkerList) return (const LOTMarkerList*)animation->mMarkerList;
+
+   animation->mMarkerList = new LOTMarkerList();
+   animation->mMarkerList->size = markers.size();
+   animation->mMarkerList->ptr = new LOTMarker[markers.size()]();
+
+   for(size_t i = 0; i < markers.size(); i++) {
+       animation->mMarkerList->ptr[i].name = strdup(std::get<0>(markers[i]).c_str());
+       animation->mMarkerList->ptr[i].startframe= std::get<1>(markers[i]);
+       animation->mMarkerList->ptr[i].endframe= std::get<2>(markers[i]);
+   }
+   return (const LOTMarkerList*)animation->mMarkerList;
+}
+
+RLOTTIE_API void
+lottie_configure_model_cache_size(size_t cacheSize)
+{
+   rlottie::configureModelCacheSize(cacheSize);
+}
+
+}

vendor/github.com/Benau/go_rlottie/config.h

@@ -0,0 +1,10 @@
+#ifndef GO_RLOTTIE_HPP
+#define GO_RLOTTIE_HPP
+#ifndef __APPLE__
+#ifdef __ARM_NEON__
+#define USE_ARM_NEON
+#endif
+#endif
+#define LOTTIE_THREAD_SUPPORT
+#define LOTTIE_CACHE_SUPPORT
+#endif

vendor/github.com/Benau/go_rlottie/generate_from_rlottie.py

@@ -0,0 +1,122 @@
+#!/usr/bin/python3
+# ./generate_from_rlottie.py /path/to/clean/rlottie/src/ /path/to/clean/rlottie/inc/
+import glob
+import os
+import re
+import sys
+
+FILE_KEYS = {}
+
+def get_closest_local_header(header):
+    for full_path, local in FILE_KEYS.items():
+        if os.path.basename(full_path) == header:
+            return local
+    return ''
+
+def fix_headers(code_text):
+    out = ''
+    has_neon = False
+    for line in code_text:
+        # Special fixes
+        if line == '#include <vpoint.h>':
+            line = '#include "vpoint.h"'
+        if line == '#include <vsharedptr.h>':
+            line = '#include "vsharedptr.h"'
+        if line == '#include <vglobal.h>':
+            line = '#include "vglobal.h"'
+        if line == '#include <vrect.h>':
+            line = '#include "vrect.h"'
+        # ARM on apple fixes
+        if '__ARM_NEON__' in line:
+            has_neon = True
+            line = line.replace('__ARM_NEON__', 'USE_ARM_NEON')
+        header_file = re.match('#include\s+["]([^"]+)["].*', line)
+        # regex to search for <, > too
+        #header_file = re.match('#include\s+[<"]([^>"]+)[>"].*', line)
+        if header_file:
+            header = header_file.groups()[0]
+            abs_header = os.path.abspath(header)
+            header_exists = os.path.exists(abs_header)
+            if header_exists and abs_header in FILE_KEYS:
+                out += '#include "' + FILE_KEYS[abs_header] + '"\n'
+            else:
+                local = get_closest_local_header(header)
+                if local != '':
+                    out += '#include "' + local + '"\n'
+                else:
+                    out += line + '\n'
+        else:
+            out += line + '\n'
+    if has_neon:
+        out = '#include "config.h"\n' + out
+    return out
+
+if len(sys.argv) < 2:
+    print('usage: ./generate_from_rlottie.py /path/to/clean/rlottie/src/ /path/to/clean/rlottie/inc/')
+    os._exit(1)
+
+code = ['.c', '.s', '.S', '.sx', 'cc', 'cpp', 'cpp' ]
+header = ['.h', '.hh', '.hpp', '.hxx' ]
+
+# Remove old files
+files = os.listdir('.')
+for file in files:
+    if file.endswith(tuple(code)) or file.endswith(tuple(header)):
+        os.remove(os.path.join('.', file))
+
+paths = []
+it = iter(sys.argv)
+next(it, None)
+for argv in it:
+    paths.append(argv)
+
+for path in paths:
+    for file in glob.iglob(path + '/**', recursive=True):
+        # Ignore msvc config.h and wasm file
+        if file.endswith('config.h') or 'wasm' in file:
+            continue
+        if file.endswith(tuple(code)) or file.endswith(tuple(header)):
+            key = os.path.abspath(file)
+            val = file.replace(path, '').replace('/', '_')
+            FILE_KEYS[key] = val
+
+header_check = []
+for full_path, local in FILE_KEYS.items():
+    header_file = os.path.basename(full_path)
+    if header_file.endswith(tuple(code)):
+        continue
+    if not header_file in header_check:
+        header_check.append(header_file)
+    else:
+        print('WARNING: ' + header_file + ' has multiple reference in subdirectories')
+
+cur_dir = os.path.abspath('.')
+for full_path, local in FILE_KEYS.items():
+    os.chdir(os.path.dirname(full_path))
+    with open(full_path) as code:
+        code_text = code.read().splitlines() 
+    code.close()
+    fixed = fix_headers(code_text)
+    os.chdir(cur_dir)
+    local_file = open(local, "w")
+    local_file.write(fixed)
+    local_file.close()
+
+# Write config.h
+config = '#ifndef GO_RLOTTIE_HPP\n#define GO_RLOTTIE_HPP\n'
+# ARM on apple won't compile
+config += '#ifndef __APPLE__\n#ifdef __ARM_NEON__\n#define USE_ARM_NEON\n#endif\n#endif\n'
+config += '#define LOTTIE_THREAD_SUPPORT\n#define LOTTIE_CACHE_SUPPORT\n'
+config += '#endif\n'
+config_file = open('config.h', "w")
+config_file.write(config)
+config_file.close()
+
+# Fix vector_pixman_pixman-arm-neon-asm.S
+with open('vector_pixman_pixman-arm-neon-asm.S') as code:
+    assembly = code.read()
+code.close()
+assembly = '#include "config.h"\n#ifdef USE_ARM_NEON\n' + assembly + '#endif\n'
+fixed_assembly = open('vector_pixman_pixman-arm-neon-asm.S', "w")
+fixed_assembly.write(assembly)
+fixed_assembly.close()

vendor/github.com/Benau/go_rlottie/go_rlottie.go

@@ -0,0 +1,56 @@
+package go_rlottie
+
+/*
+#cgo !windows LDFLAGS: -lm
+#cgo windows CFLAGS: -DRLOTTIE_BUILD=0
+#cgo windows CXXFLAGS: -DRLOTTIE_BUILD=0
+#cgo CXXFLAGS: -std=c++14 -fno-exceptions -fno-asynchronous-unwind-tables -fno-rtti -Wall -fvisibility=hidden -Wnon-virtual-dtor -Woverloaded-virtual -Wno-unused-parameter
+#include "rlottie_capi.h"
+void lottie_configure_model_cache_size(size_t cacheSize);
+*/
+import "C"
+import "unsafe"
+
+type Lottie_Animation *C.Lottie_Animation
+
+func LottieConfigureModelCacheSize(size uint) {
+	C.lottie_configure_model_cache_size(C.size_t(size))
+}
+
+func LottieAnimationFromData(data string, key string, resource_path string) Lottie_Animation {
+	var animation Lottie_Animation
+	animation = C.lottie_animation_from_data(C.CString(data), C.CString(key), C.CString(resource_path))
+	return animation
+}
+
+func LottieAnimationDestroy(animation Lottie_Animation) {
+	C.lottie_animation_destroy(animation)
+}
+
+func LottieAnimationGetSize(animation Lottie_Animation) (uint, uint) {
+	var width C.size_t
+	var height C.size_t
+	C.lottie_animation_get_size(animation, &width, &height)
+	return uint(width), uint(height)
+}
+
+func LottieAnimationGetTotalframe(animation Lottie_Animation) uint {
+	return uint(C.lottie_animation_get_totalframe(animation))
+}
+
+func LottieAnimationGetFramerate(animation Lottie_Animation) float64 {
+	return float64(C.lottie_animation_get_framerate(animation))
+}
+
+func LottieAnimationGetFrameAtPos(animation Lottie_Animation, pos float32) uint {
+	return uint(C.lottie_animation_get_frame_at_pos(animation, C.float(pos)))
+}
+
+func LottieAnimationGetDuration(animation Lottie_Animation) float64 {
+	return float64(C.lottie_animation_get_duration(animation))
+}
+
+func LottieAnimationRender(animation Lottie_Animation, frame_num uint, buffer []byte, width uint, height uint, bytes_per_line uint) {
+	var ptr *C.uint32_t = (*C.uint32_t)(unsafe.Pointer(&buffer[0]));
+	C.lottie_animation_render(animation, C.size_t(frame_num), ptr, C.size_t(width), C.size_t(height), C.size_t(bytes_per_line))
+}

vendor/github.com/Benau/go_rlottie/lottie_lottieanimation.cpp

@@ -0,0 +1,457 @@
+#include "config.h"
+/*
+ * Copyright (c) 2020 Samsung Electronics Co., Ltd. All rights reserved.
+
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include "config.h"
+#include "lottie_lottieitem.h"
+#include "lottie_lottiemodel.h"
+#include "rlottie.h"
+
+#include <fstream>
+
+using namespace rlottie;
+using namespace rlottie::internal;
+
+RLOTTIE_API void rlottie::configureModelCacheSize(size_t cacheSize)
+{
+    internal::model::configureModelCacheSize(cacheSize);
+}
+
+struct RenderTask {
+    RenderTask() { receiver = sender.get_future(); }
+    std::promise<Surface> sender;
+    std::future<Surface>  receiver;
+    AnimationImpl *       playerImpl{nullptr};
+    size_t                frameNo{0};
+    Surface               surface;
+    bool                  keepAspectRatio{true};
+};
+using SharedRenderTask = std::shared_ptr<RenderTask>;
+
+class AnimationImpl {
+public:
+    void    init(std::shared_ptr<model::Composition> composition);
+    bool    update(size_t frameNo, const VSize &size, bool keepAspectRatio);
+    VSize   size() const { return mModel->size(); }
+    double  duration() const { return mModel->duration(); }
+    double  frameRate() const { return mModel->frameRate(); }
+    size_t  totalFrame() const { return mModel->totalFrame(); }
+    size_t  frameAtPos(double pos) const { return mModel->frameAtPos(pos); }
+    Surface render(size_t frameNo, const Surface &surface,
+                   bool keepAspectRatio);
+    std::future<Surface> renderAsync(size_t frameNo, Surface &&surface,
+                                     bool keepAspectRatio);
+    const LOTLayerNode * renderTree(size_t frameNo, const VSize &size);
+
+    const LayerInfoList &layerInfoList() const
+    {
+        if (mLayerList.empty()) {
+            mLayerList = mModel->layerInfoList();
+        }
+        return mLayerList;
+    }
+    const MarkerList &markers() const { return mModel->markers(); }
+    void              setValue(const std::string &keypath, LOTVariant &&value);
+    void              removeFilter(const std::string &keypath, Property prop);
+
+private:
+    mutable LayerInfoList                  mLayerList;
+    model::Composition *                   mModel;
+    SharedRenderTask                       mTask;
+    std::atomic<bool>                      mRenderInProgress;
+    std::unique_ptr<renderer::Composition> mRenderer{nullptr};
+};
+
+void AnimationImpl::setValue(const std::string &keypath, LOTVariant &&value)
+{
+    if (keypath.empty()) return;
+    mRenderer->setValue(keypath, value);
+}
+
+const LOTLayerNode *AnimationImpl::renderTree(size_t frameNo, const VSize &size)
+{
+    if (update(frameNo, size, true)) {
+        mRenderer->buildRenderTree();
+    }
+    return mRenderer->renderTree();
+}
+
+bool AnimationImpl::update(size_t frameNo, const VSize &size,
+                           bool keepAspectRatio)
+{
+    frameNo += mModel->startFrame();
+
+    if (frameNo > mModel->endFrame()) frameNo = mModel->endFrame();
+
+    if (frameNo < mModel->startFrame()) frameNo = mModel->startFrame();
+
+    return mRenderer->update(int(frameNo), size, keepAspectRatio);
+}
+
+Surface AnimationImpl::render(size_t frameNo, const Surface &surface,
+                              bool keepAspectRatio)
+{
+    bool renderInProgress = mRenderInProgress.load();
+    if (renderInProgress) {
+        vCritical << "Already Rendering Scheduled for this Animation";
+        return surface;
+    }
+
+    mRenderInProgress.store(true);
+    update(
+        frameNo,
+        VSize(int(surface.drawRegionWidth()), int(surface.drawRegionHeight())),
+        keepAspectRatio);
+    mRenderer->render(surface);
+    mRenderInProgress.store(false);
+
+    return surface;
+}
+
+void AnimationImpl::init(std::shared_ptr<model::Composition> composition)
+{
+    mModel = composition.get();
+    mRenderer = std::make_unique<renderer::Composition>(composition);
+    mRenderInProgress = false;
+}
+
+#ifdef LOTTIE_THREAD_SUPPORT
+
+#include <thread>
+#include "vector_vtaskqueue.h"
+
+/*
+ * Implement a task stealing schduler to perform render task
+ * As each player draws into its own buffer we can delegate this
+ * task to a slave thread. The scheduler creates a threadpool depending
+ * on the number of cores available in the system and does a simple fair
+ * scheduling by assigning the task in a round-robin fashion. Each thread
+ * in the threadpool has its own queue. once it finishes all the task on its
+ * own queue it goes through rest of the queue and looks for task if it founds
+ * one it steals the task from it and executes. if it couldn't find one then it
+ * just waits for new task on its own queue.
+ */
+class RenderTaskScheduler {
+    const unsigned           _count{std::thread::hardware_concurrency()};
+    std::vector<std::thread> _threads;
+    std::vector<TaskQueue<SharedRenderTask>> _q{_count};
+    std::atomic<unsigned>                    _index{0};
+
+    void run(unsigned i)
+    {
+        while (true) {
+            bool             success = false;
+            SharedRenderTask task;
+            for (unsigned n = 0; n != _count * 2; ++n) {
+                if (_q[(i + n) % _count].try_pop(task)) {
+                    success = true;
+                    break;
+                }
+            }
+            if (!success && !_q[i].pop(task)) break;
+
+            auto result = task->playerImpl->render(task->frameNo, task->surface,
+                                                   task->keepAspectRatio);
+            task->sender.set_value(result);
+        }
+    }
+
+    RenderTaskScheduler()
+    {
+        for (unsigned n = 0; n != _count; ++n) {
+            _threads.emplace_back([&, n] { run(n); });
+        }
+    }
+
+public:
+    static RenderTaskScheduler &instance()
+    {
+        static RenderTaskScheduler singleton;
+        return singleton;
+    }
+
+    ~RenderTaskScheduler()
+    {
+        for (auto &e : _q) e.done();
+
+        for (auto &e : _threads) e.join();
+    }
+
+    std::future<Surface> process(SharedRenderTask task)
+    {
+        auto receiver = std::move(task->receiver);
+        auto i = _index++;
+
+        for (unsigned n = 0; n != _count; ++n) {
+            if (_q[(i + n) % _count].try_push(std::move(task))) return receiver;
+        }
+
+        if (_count > 0) {
+            _q[i % _count].push(std::move(task));
+        }
+
+        return receiver;
+    }
+};
+
+#else
+class RenderTaskScheduler {
+public:
+    static RenderTaskScheduler &instance()
+    {
+        static RenderTaskScheduler singleton;
+        return singleton;
+    }
+
+    std::future<Surface> process(SharedRenderTask task)
+    {
+        auto result = task->playerImpl->render(task->frameNo, task->surface,
+                                               task->keepAspectRatio);
+        task->sender.set_value(result);
+        return std::move(task->receiver);
+    }
+};
+#endif
+
+std::future<Surface> AnimationImpl::renderAsync(size_t    frameNo,
+                                                Surface &&surface,
+                                                bool      keepAspectRatio)
+{
+    if (!mTask) {
+        mTask = std::make_shared<RenderTask>();
+    } else {
+        mTask->sender = std::promise<Surface>();
+        mTask->receiver = mTask->sender.get_future();
+    }
+    mTask->playerImpl = this;
+    mTask->frameNo = frameNo;
+    mTask->surface = std::move(surface);
+    mTask->keepAspectRatio = keepAspectRatio;
+
+    return RenderTaskScheduler::instance().process(mTask);
+}
+
+/**
+ * \breif Brief abput the Api.
+ * Description about the setFilePath Api
+ * @param path  add the details
+ */
+std::unique_ptr<Animation> Animation::loadFromData(
+    std::string jsonData, const std::string &key,
+    const std::string &resourcePath, bool cachePolicy)
+{
+    if (jsonData.empty()) {
+        vWarning << "jason data is empty";
+        return nullptr;
+    }
+
+    auto composition = model::loadFromData(std::move(jsonData), key,
+                                           resourcePath, cachePolicy);
+    if (composition) {
+        auto animation = std::unique_ptr<Animation>(new Animation);
+        animation->d->init(std::move(composition));
+        return animation;
+    }
+
+    return nullptr;
+}
+
+std::unique_ptr<Animation> Animation::loadFromData(std::string jsonData,
+                                                   std::string resourcePath,
+                                                   ColorFilter filter)
+{
+    if (jsonData.empty()) {
+        vWarning << "jason data is empty";
+        return nullptr;
+    }
+
+    auto composition = model::loadFromData(
+        std::move(jsonData), std::move(resourcePath), std::move(filter));
+    if (composition) {
+        auto animation = std::unique_ptr<Animation>(new Animation);
+        animation->d->init(std::move(composition));
+        return animation;
+    }
+    return nullptr;
+}
+
+std::unique_ptr<Animation> Animation::loadFromFile(const std::string &path,
+                                                   bool cachePolicy)
+{
+    if (path.empty()) {
+        vWarning << "File path is empty";
+        return nullptr;
+    }
+
+    auto composition = model::loadFromFile(path, cachePolicy);
+    if (composition) {
+        auto animation = std::unique_ptr<Animation>(new Animation);
+        animation->d->init(std::move(composition));
+        return animation;
+    }
+    return nullptr;
+}
+
+void Animation::size(size_t &width, size_t &height) const
+{
+    VSize sz = d->size();
+
+    width = sz.width();
+    height = sz.height();
+}
+
+double Animation::duration() const
+{
+    return d->duration();
+}
+
+double Animation::frameRate() const
+{
+    return d->frameRate();
+}
+
+size_t Animation::totalFrame() const
+{
+    return d->totalFrame();
+}
+
+size_t Animation::frameAtPos(double pos)
+{
+    return d->frameAtPos(pos);
+}
+
+const LOTLayerNode *Animation::renderTree(size_t frameNo, size_t width,
+                                          size_t height) const
+{
+    return d->renderTree(frameNo, VSize(int(width), int(height)));
+}
+
+std::future<Surface> Animation::render(size_t frameNo, Surface surface,
+                                       bool keepAspectRatio)
+{
+    return d->renderAsync(frameNo, std::move(surface), keepAspectRatio);
+}
+
+void Animation::renderSync(size_t frameNo, Surface surface,
+                           bool keepAspectRatio)
+{
+    d->render(frameNo, surface, keepAspectRatio);
+}
+
+const LayerInfoList &Animation::layers() const
+{
+    return d->layerInfoList();
+}
+
+const MarkerList &Animation::markers() const
+{
+    return d->markers();
+}
+
+void Animation::setValue(Color_Type, Property prop, const std::string &keypath,
+                         Color value)
+{
+    d->setValue(keypath,
+                LOTVariant(prop, [value](const FrameInfo &) { return value; }));
+}
+
+void Animation::setValue(Float_Type, Property prop, const std::string &keypath,
+                         float value)
+{
+    d->setValue(keypath,
+                LOTVariant(prop, [value](const FrameInfo &) { return value; }));
+}
+
+void Animation::setValue(Size_Type, Property prop, const std::string &keypath,
+                         Size value)
+{
+    d->setValue(keypath,
+                LOTVariant(prop, [value](const FrameInfo &) { return value; }));
+}
+
+void Animation::setValue(Point_Type, Property prop, const std::string &keypath,
+                         Point value)
+{
+    d->setValue(keypath,
+                LOTVariant(prop, [value](const FrameInfo &) { return value; }));
+}
+
+void Animation::setValue(Color_Type, Property prop, const std::string &keypath,
+                         std::function<Color(const FrameInfo &)> &&value)
+{
+    d->setValue(keypath, LOTVariant(prop, value));
+}
+
+void Animation::setValue(Float_Type, Property prop, const std::string &keypath,
+                         std::function<float(const FrameInfo &)> &&value)
+{
+    d->setValue(keypath, LOTVariant(prop, value));
+}
+
+void Animation::setValue(Size_Type, Property prop, const std::string &keypath,
+                         std::function<Size(const FrameInfo &)> &&value)
+{
+    d->setValue(keypath, LOTVariant(prop, value));
+}
+
+void Animation::setValue(Point_Type, Property prop, const std::string &keypath,
+                         std::function<Point(const FrameInfo &)> &&value)
+{
+    d->setValue(keypath, LOTVariant(prop, value));
+}
+
+Animation::~Animation() = default;
+Animation::Animation() : d(std::make_unique<AnimationImpl>()) {}
+
+Surface::Surface(uint32_t *buffer, size_t width, size_t height,
+                 size_t bytesPerLine)
+    : mBuffer(buffer),
+      mWidth(width),
+      mHeight(height),
+      mBytesPerLine(bytesPerLine)
+{
+    mDrawArea.w = mWidth;
+    mDrawArea.h = mHeight;
+}
+
+void Surface::setDrawRegion(size_t x, size_t y, size_t width, size_t height)
+{
+    if ((x + width > mWidth) || (y + height > mHeight)) return;
+
+    mDrawArea.x = x;
+    mDrawArea.y = y;
+    mDrawArea.w = width;
+    mDrawArea.h = height;
+}
+
+#ifdef LOTTIE_LOGGING_SUPPORT
+void initLogging()
+{
+#if defined(USE_ARM_NEON)
+    set_log_level(LogLevel::OFF);
+#else
+    initialize(GuaranteedLogger(), "/tmp/", "rlottie", 1);
+    set_log_level(LogLevel::INFO);
+#endif
+}
+
+V_CONSTRUCTOR_FUNCTION(initLogging)
+#endif

vendor/github.com/Benau/go_rlottie/lottie_lottiefiltermodel.h

@@ -0,0 +1,435 @@
+/*
+ * Copyright (c) 2020 Samsung Electronics Co., Ltd. All rights reserved.
+
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef LOTTIEFILTERMODEL_H
+#define LOTTIEFILTERMODEL_H
+
+#include <algorithm>
+#include <bitset>
+#include <cassert>
+#include "lottie_lottiemodel.h"
+#include "rlottie.h"
+
+using namespace rlottie::internal;
+// Naive way to implement std::variant
+// refactor it when we move to c++17
+// users should make sure proper combination
+// of id and value are passed while creating the object.
+class LOTVariant {
+public:
+    using ValueFunc = std::function<float(const rlottie::FrameInfo&)>;
+    using ColorFunc = std::function<rlottie::Color(const rlottie::FrameInfo&)>;
+    using PointFunc = std::function<rlottie::Point(const rlottie::FrameInfo&)>;
+    using SizeFunc = std::function<rlottie::Size(const rlottie::FrameInfo&)>;
+
+    LOTVariant(rlottie::Property prop, const ValueFunc& v)
+        : mPropery(prop), mTag(Value)
+    {
+        construct(impl.valueFunc, v);
+    }
+
+    LOTVariant(rlottie::Property prop, ValueFunc&& v)
+        : mPropery(prop), mTag(Value)
+    {
+        moveConstruct(impl.valueFunc, std::move(v));
+    }
+
+    LOTVariant(rlottie::Property prop, const ColorFunc& v)
+        : mPropery(prop), mTag(Color)
+    {
+        construct(impl.colorFunc, v);
+    }
+
+    LOTVariant(rlottie::Property prop, ColorFunc&& v)
+        : mPropery(prop), mTag(Color)
+    {
+        moveConstruct(impl.colorFunc, std::move(v));
+    }
+
+    LOTVariant(rlottie::Property prop, const PointFunc& v)
+        : mPropery(prop), mTag(Point)
+    {
+        construct(impl.pointFunc, v);
+    }
+
+    LOTVariant(rlottie::Property prop, PointFunc&& v)
+        : mPropery(prop), mTag(Point)
+    {
+        moveConstruct(impl.pointFunc, std::move(v));
+    }
+
+    LOTVariant(rlottie::Property prop, const SizeFunc& v)
+        : mPropery(prop), mTag(Size)
+    {
+        construct(impl.sizeFunc, v);
+    }
+
+    LOTVariant(rlottie::Property prop, SizeFunc&& v)
+        : mPropery(prop), mTag(Size)
+    {
+        moveConstruct(impl.sizeFunc, std::move(v));
+    }
+
+    rlottie::Property property() const { return mPropery; }
+
+    const ColorFunc& color() const
+    {
+        assert(mTag == Color);
+        return impl.colorFunc;
+    }
+
+    const ValueFunc& value() const
+    {
+        assert(mTag == Value);
+        return impl.valueFunc;
+    }
+
+    const PointFunc& point() const
+    {
+        assert(mTag == Point);
+        return impl.pointFunc;
+    }
+
+    const SizeFunc& size() const
+    {
+        assert(mTag == Size);
+        return impl.sizeFunc;
+    }
+
+    LOTVariant() = default;
+    ~LOTVariant() noexcept { Destroy(); }
+    LOTVariant(const LOTVariant& other) { Copy(other); }
+    LOTVariant(LOTVariant&& other) noexcept { Move(std::move(other)); }
+    LOTVariant& operator=(LOTVariant&& other)
+    {
+        Destroy();
+        Move(std::move(other));
+        return *this;
+    }
+    LOTVariant& operator=(const LOTVariant& other)
+    {
+        Destroy();
+        Copy(other);
+        return *this;
+    }
+
+private:
+    template <typename T>
+    void construct(T& member, const T& val)
+    {
+        new (&member) T(val);
+    }
+
+    template <typename T>
+    void moveConstruct(T& member, T&& val)
+    {
+        new (&member) T(std::move(val));
+    }
+
+    void Move(LOTVariant&& other)
+    {
+        switch (other.mTag) {
+        case Type::Value:
+            moveConstruct(impl.valueFunc, std::move(other.impl.valueFunc));
+            break;
+        case Type::Color:
+            moveConstruct(impl.colorFunc, std::move(other.impl.colorFunc));
+            break;
+        case Type::Point:
+            moveConstruct(impl.pointFunc, std::move(other.impl.pointFunc));
+            break;
+        case Type::Size:
+            moveConstruct(impl.sizeFunc, std::move(other.impl.sizeFunc));
+            break;
+        default:
+            break;
+        }
+        mTag = other.mTag;
+        mPropery = other.mPropery;
+        other.mTag = MonoState;
+    }
+
+    void Copy(const LOTVariant& other)
+    {
+        switch (other.mTag) {
+        case Type::Value:
+            construct(impl.valueFunc, other.impl.valueFunc);
+            break;
+        case Type::Color:
+            construct(impl.colorFunc, other.impl.colorFunc);
+            break;
+        case Type::Point:
+            construct(impl.pointFunc, other.impl.pointFunc);
+            break;
+        case Type::Size:
+            construct(impl.sizeFunc, other.impl.sizeFunc);
+            break;
+        default:
+            break;
+        }
+        mTag = other.mTag;
+        mPropery = other.mPropery;
+    }
+
+    void Destroy()
+    {
+        switch (mTag) {
+        case MonoState: {
+            break;
+        }
+        case Value: {
+            impl.valueFunc.~ValueFunc();
+            break;
+        }
+        case Color: {
+            impl.colorFunc.~ColorFunc();
+            break;
+        }
+        case Point: {
+            impl.pointFunc.~PointFunc();
+            break;
+        }
+        case Size: {
+            impl.sizeFunc.~SizeFunc();
+            break;
+        }
+        }
+    }
+
+    enum Type { MonoState, Value, Color, Point, Size };
+    rlottie::Property mPropery;
+    Type              mTag{MonoState};
+    union details {
+        ColorFunc colorFunc;
+        ValueFunc valueFunc;
+        PointFunc pointFunc;
+        SizeFunc  sizeFunc;
+        details() {}
+        ~details() noexcept {}
+    } impl;
+};
+
+namespace rlottie {
+
+namespace internal {
+
+namespace model {
+
+class FilterData {
+public:
+    void addValue(LOTVariant& value)
+    {
+        uint index = static_cast<uint>(value.property());
+        if (mBitset.test(index)) {
+            std::replace_if(mFilters.begin(), mFilters.end(),
+                            [&value](const LOTVariant& e) {
+                                return e.property() == value.property();
+                            },
+                            value);
+        } else {
+            mBitset.set(index);
+            mFilters.push_back(value);
+        }
+    }
+
+    void removeValue(LOTVariant& value)
+    {
+        uint index = static_cast<uint>(value.property());
+        if (mBitset.test(index)) {
+            mBitset.reset(index);
+            mFilters.erase(std::remove_if(mFilters.begin(), mFilters.end(),
+                                          [&value](const LOTVariant& e) {
+                                              return e.property() ==
+                                                     value.property();
+                                          }),
+                           mFilters.end());
+        }
+    }
+    bool hasFilter(rlottie::Property prop) const
+    {
+        return mBitset.test(static_cast<uint>(prop));
+    }
+    model::Color color(rlottie::Property prop, int frame) const
+    {
+        rlottie::FrameInfo info(frame);
+        rlottie::Color     col = data(prop).color()(info);
+        return model::Color(col.r(), col.g(), col.b());
+    }
+    VPointF point(rlottie::Property prop, int frame) const
+    {
+        rlottie::FrameInfo info(frame);
+        rlottie::Point     pt = data(prop).point()(info);
+        return VPointF(pt.x(), pt.y());
+    }
+    VSize scale(rlottie::Property prop, int frame) const
+    {
+        rlottie::FrameInfo info(frame);
+        rlottie::Size      sz = data(prop).size()(info);
+        return VSize(sz.w(), sz.h());
+    }
+    float opacity(rlottie::Property prop, int frame) const
+    {
+        rlottie::FrameInfo info(frame);
+        float              val = data(prop).value()(info);
+        return val / 100;
+    }
+    float value(rlottie::Property prop, int frame) const
+    {
+        rlottie::FrameInfo info(frame);
+        return data(prop).value()(info);
+    }
+
+private:
+    const LOTVariant& data(rlottie::Property prop) const
+    {
+        auto result = std::find_if(
+            mFilters.begin(), mFilters.end(),
+            [prop](const LOTVariant& e) { return e.property() == prop; });
+        return *result;
+    }
+    std::bitset<32>         mBitset{0};
+    std::vector<LOTVariant> mFilters;
+};
+
+template <typename T>
+struct FilterBase
+{
+    FilterBase(T *model): model_(model){}
+
+    const char*  name() const { return model_->name(); }
+
+    FilterData* filter() {
+        if (!filterData_) filterData_ = std::make_unique<FilterData>();
+        return filterData_.get();
+    }
+
+    const FilterData * filter() const { return filterData_.get(); }
+    const T* model() const { return model_;}
+
+    bool hasFilter(rlottie::Property prop) const {
+        return filterData_ ? filterData_->hasFilter(prop)
+                         : false;
+    }
+
+    T*                           model_{nullptr};
+    std::unique_ptr<FilterData>  filterData_{nullptr};
+};
+
+
+template <typename T>
+class Filter : public FilterBase<T> {
+public:
+    Filter(T* model): FilterBase<T>(model){}
+    model::Color color(int frame) const
+    {
+        if (this->hasFilter(rlottie::Property::StrokeColor)) {
+            return this->filter()->color(rlottie::Property::StrokeColor, frame);
+        }
+        return this->model()->color(frame);
+    }
+    float opacity(int frame) const
+    {
+        if (this->hasFilter(rlottie::Property::StrokeOpacity)) {
+            return this->filter()->opacity(rlottie::Property::StrokeOpacity, frame);
+        }
+        return this->model()->opacity(frame);
+    }
+
+    float strokeWidth(int frame) const
+    {
+        if (this->hasFilter(rlottie::Property::StrokeWidth)) {
+            return this->filter()->value(rlottie::Property::StrokeWidth, frame);
+        }
+        return this->model()->strokeWidth(frame);
+    }
+
+    float     miterLimit() const { return this->model()->miterLimit(); }
+    CapStyle  capStyle() const { return this->model()->capStyle(); }
+    JoinStyle joinStyle() const { return this->model()->joinStyle(); }
+    bool      hasDashInfo() const { return this->model()->hasDashInfo(); }
+    void      getDashInfo(int frameNo, std::vector<float>& result) const
+    {
+        return this->model()->getDashInfo(frameNo, result);
+    }
+};
+
+
+template <>
+class Filter<model::Fill>: public FilterBase<model::Fill>
+{
+public:
+    Filter(model::Fill* model) : FilterBase<model::Fill>(model) {}
+
+    model::Color color(int frame) const
+    {
+        if (this->hasFilter(rlottie::Property::FillColor)) {
+            return this->filter()->color(rlottie::Property::FillColor, frame);
+        }
+        return this->model()->color(frame);
+    }
+
+    float opacity(int frame) const
+    {
+        if (this->hasFilter(rlottie::Property::FillOpacity)) {
+            return this->filter()->opacity(rlottie::Property::FillOpacity, frame);
+        }
+        return this->model()->opacity(frame);
+    }
+
+    FillRule fillRule() const { return this->model()->fillRule(); }
+};
+
+template <>
+class Filter<model::Group> : public FilterBase<model::Group>
+{
+public:
+    Filter(model::Group* model = nullptr) : FilterBase<model::Group>(model) {}
+
+    bool   hasModel() const { return this->model() ? true : false; }
+
+    model::Transform* transform() const { return this->model() ? this->model()->mTransform : nullptr; }
+    VMatrix           matrix(int frame) const
+    {
+        VMatrix mS, mR, mT;
+        if (this->hasFilter(rlottie::Property::TrScale)) {
+            VSize s = this->filter()->scale(rlottie::Property::TrScale, frame);
+            mS.scale(s.width() / 100.0, s.height() / 100.0);
+        }
+        if (this->hasFilter(rlottie::Property::TrRotation)) {
+            mR.rotate(this->filter()->value(rlottie::Property::TrRotation, frame));
+        }
+        if (this->hasFilter(rlottie::Property::TrPosition)) {
+            mT.translate(this->filter()->point(rlottie::Property::TrPosition, frame));
+        }
+
+        return this->model()->mTransform->matrix(frame) * mS * mR * mT;
+    }
+};
+
+
+}  // namespace model
+
+}  // namespace internal
+
+}  // namespace rlottie
+
+#endif  // LOTTIEFILTERMODEL_H

vendor/github.com/Benau/go_rlottie/lottie_lottieitem.h

@@ -0,0 +1,626 @@
+/*
+ * Copyright (c) 2020 Samsung Electronics Co., Ltd. All rights reserved.
+
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef LOTTIEITEM_H
+#define LOTTIEITEM_H
+
+#include <memory>
+#include <sstream>
+
+#include "lottie_lottiekeypath.h"
+#include "lottie_lottiefiltermodel.h"
+#include "rlottie.h"
+#include "rlottiecommon.h"
+#include "vector_varenaalloc.h"
+#include "vector_vdrawable.h"
+#include "vector_vmatrix.h"
+#include "vector_vpainter.h"
+#include "vector_vpath.h"
+#include "vector_vpathmesure.h"
+#include "vector_vpoint.h"
+
+V_USE_NAMESPACE
+
+namespace rlottie {
+
+namespace internal {
+
+template <class T>
+class VSpan {
+public:
+    using reference = T &;
+    using pointer = T *;
+    using const_pointer = T const *;
+    using const_reference = T const &;
+    using index_type = size_t;
+
+    using iterator = pointer;
+    using const_iterator = const_pointer;
+
+    VSpan() = default;
+    VSpan(pointer data, index_type size) : _data(data), _size(size) {}
+
+    constexpr pointer        data() const noexcept { return _data; }
+    constexpr index_type     size() const noexcept { return _size; }
+    constexpr bool           empty() const noexcept { return size() == 0; }
+    constexpr iterator       begin() const noexcept { return data(); }
+    constexpr iterator       end() const noexcept { return data() + size(); }
+    constexpr const_iterator cbegin() const noexcept { return data(); }
+    constexpr const_iterator cend() const noexcept { return data() + size(); }
+    constexpr reference      operator[](index_type idx) const
+    {
+        return *(data() + idx);
+    }
+
+private:
+    pointer    _data{nullptr};
+    index_type _size{0};
+};
+
+namespace renderer {
+
+using DrawableList = VSpan<VDrawable *>;
+
+enum class DirtyFlagBit : uchar {
+    None = 0x00,
+    Matrix = 0x01,
+    Alpha = 0x02,
+    All = (Matrix | Alpha)
+};
+typedef vFlag<DirtyFlagBit> DirtyFlag;
+
+class SurfaceCache {
+public:
+    SurfaceCache() { mCache.reserve(10); }
+
+    VBitmap make_surface(
+        size_t width, size_t height,
+        VBitmap::Format format = VBitmap::Format::ARGB32_Premultiplied)
+    {
+        if (mCache.empty()) return {width, height, format};
+
+        auto surface = mCache.back();
+        surface.reset(width, height, format);
+
+        mCache.pop_back();
+        return surface;
+    }
+
+    void release_surface(VBitmap &surface) { mCache.push_back(surface); }
+
+private:
+    std::vector<VBitmap> mCache;
+};
+
+class Drawable final : public VDrawable {
+public:
+    void sync();
+
+public:
+    std::unique_ptr<LOTNode> mCNode{nullptr};
+
+    ~Drawable() noexcept
+    {
+        if (mCNode && mCNode->mGradient.stopPtr)
+            free(mCNode->mGradient.stopPtr);
+    }
+};
+
+struct CApiData {
+    CApiData();
+    LOTLayerNode                mLayer;
+    std::vector<LOTMask>        mMasks;
+    std::vector<LOTLayerNode *> mLayers;
+    std::vector<LOTNode *>      mCNodeList;
+};
+
+class Clipper {
+public:
+    explicit Clipper(VSize size) : mSize(size) {}
+    void update(const VMatrix &matrix);
+    void preprocess(const VRect &clip);
+    VRle rle(const VRle &mask);
+
+public:
+    VSize       mSize;
+    VPath       mPath;
+    VRle        mMaskedRle;
+    VRasterizer mRasterizer;
+    bool        mRasterRequest{false};
+};
+
+class Mask {
+public:
+    explicit Mask(model::Mask *data) : mData(data) {}
+    void update(int frameNo, const VMatrix &parentMatrix, float parentAlpha,
+                const DirtyFlag &flag);
+    model::Mask::Mode maskMode() const { return mData->mMode; }
+    VRle              rle();
+    void              preprocess(const VRect &clip);
+    bool              inverted() const { return mData->mInv; }
+public:
+    model::Mask *mData{nullptr};
+    VPath        mLocalPath;
+    VPath        mFinalPath;
+    VRasterizer  mRasterizer;
+    float        mCombinedAlpha{0};
+    bool         mRasterRequest{false};
+};
+
+/*
+ * Handels mask property of a layer item
+ */
+class LayerMask {
+public:
+    explicit LayerMask(model::Layer *layerData);
+    void update(int frameNo, const VMatrix &parentMatrix, float parentAlpha,
+                const DirtyFlag &flag);
+    bool isStatic() const { return mStatic; }
+    VRle maskRle(const VRect &clipRect);
+    void preprocess(const VRect &clip);
+
+public:
+    std::vector<Mask> mMasks;
+    VRle              mRle;
+    bool              mStatic{true};
+    bool              mDirty{true};
+};
+
+class Layer;
+
+class Composition {
+public:
+    explicit Composition(std::shared_ptr<model::Composition> composition);
+    bool  update(int frameNo, const VSize &size, bool keepAspectRatio);
+    VSize size() const { return mViewSize; }
+    void  buildRenderTree();
+    const LOTLayerNode *renderTree() const;
+    bool                render(const rlottie::Surface &surface);
+    void                setValue(const std::string &keypath, LOTVariant &value);
+
+private:
+    SurfaceCache                        mSurfaceCache;
+    VBitmap                             mSurface;
+    VMatrix                             mScaleMatrix;
+    VSize                               mViewSize;
+    std::shared_ptr<model::Composition> mModel;
+    Layer *                             mRootLayer{nullptr};
+    VArenaAlloc                         mAllocator{2048};
+    int                                 mCurFrameNo;
+    bool                                mKeepAspectRatio{true};
+};
+
+class Layer {
+public:
+    virtual ~Layer() = default;
+    Layer &operator=(Layer &&) noexcept = delete;
+    Layer(model::Layer *layerData);
+    int          id() const { return mLayerData->id(); }
+    int          parentId() const { return mLayerData->parentId(); }
+    void         setParentLayer(Layer *parent) { mParentLayer = parent; }
+    void         setComplexContent(bool value) { mComplexContent = value; }
+    bool         complexContent() const { return mComplexContent; }
+    virtual void update(int frameNo, const VMatrix &parentMatrix,
+                        float parentAlpha);
+    VMatrix      matrix(int frameNo) const;
+    void         preprocess(const VRect &clip);
+    virtual DrawableList renderList() { return {}; }
+    virtual void         render(VPainter *painter, const VRle &mask,
+                                const VRle &matteRle, SurfaceCache &cache);
+    bool                 hasMatte()
+    {
+        if (mLayerData->mMatteType == model::MatteType::None) return false;
+        return true;
+    }
+    model::MatteType matteType() const { return mLayerData->mMatteType; }
+    bool             visible() const;
+    virtual void     buildLayerNode();
+    LOTLayerNode &   clayer() { return mCApiData->mLayer; }
+    std::vector<LOTLayerNode *> &clayers() { return mCApiData->mLayers; }
+    std::vector<LOTMask> &       cmasks() { return mCApiData->mMasks; }
+    std::vector<LOTNode *> &     cnodes() { return mCApiData->mCNodeList; }
+    const char *                 name() const { return mLayerData->name(); }
+    virtual bool                 resolveKeyPath(LOTKeyPath &keyPath, uint depth,
+                                                LOTVariant &value);
+
+protected:
+    virtual void   preprocessStage(const VRect &clip) = 0;
+    virtual void   updateContent() = 0;
+    inline VMatrix combinedMatrix() const { return mCombinedMatrix; }
+    inline int     frameNo() const { return mFrameNo; }
+    inline float   combinedAlpha() const { return mCombinedAlpha; }
+    inline bool    isStatic() const { return mLayerData->isStatic(); }
+    float opacity(int frameNo) const { return mLayerData->opacity(frameNo); }
+    inline DirtyFlag flag() const { return mDirtyFlag; }
+    bool             skipRendering() const
+    {
+        return (!visible() || vIsZero(combinedAlpha()));
+    }
+
+protected:
+    std::unique_ptr<LayerMask> mLayerMask;
+    model::Layer *             mLayerData{nullptr};
+    Layer *                    mParentLayer{nullptr};
+    VMatrix                    mCombinedMatrix;
+    float                      mCombinedAlpha{0.0};
+    int                        mFrameNo{-1};
+    DirtyFlag                  mDirtyFlag{DirtyFlagBit::All};
+    bool                       mComplexContent{false};
+    std::unique_ptr<CApiData>  mCApiData;
+};
+
+class CompLayer final : public Layer {
+public:
+    explicit CompLayer(model::Layer *layerData, VArenaAlloc *allocator);
+
+    void render(VPainter *painter, const VRle &mask, const VRle &matteRle,
+                SurfaceCache &cache) final;
+    void buildLayerNode() final;
+    bool resolveKeyPath(LOTKeyPath &keyPath, uint depth,
+                        LOTVariant &value) override;
+
+protected:
+    void preprocessStage(const VRect &clip) final;
+    void updateContent() final;
+
+private:
+    void renderHelper(VPainter *painter, const VRle &mask, const VRle &matteRle,
+                      SurfaceCache &cache);
+    void renderMatteLayer(VPainter *painter, const VRle &inheritMask,
+                          const VRle &matteRle, Layer *layer, Layer *src,
+                          SurfaceCache &cache);
+
+private:
+    std::vector<Layer *>     mLayers;
+    std::unique_ptr<Clipper> mClipper;
+};
+
+class SolidLayer final : public Layer {
+public:
+    explicit SolidLayer(model::Layer *layerData);
+    void         buildLayerNode() final;
+    DrawableList renderList() final;
+
+protected:
+    void preprocessStage(const VRect &clip) final;
+    void updateContent() final;
+
+private:
+    Drawable   mRenderNode;
+    VPath      mPath;
+    VDrawable *mDrawableList{nullptr};  // to work with the Span api
+};
+
+class Group;
+
+class ShapeLayer final : public Layer {
+public:
+    explicit ShapeLayer(model::Layer *layerData, VArenaAlloc *allocator);
+    DrawableList renderList() final;
+    void         buildLayerNode() final;
+    bool         resolveKeyPath(LOTKeyPath &keyPath, uint depth,
+                                LOTVariant &value) override;
+
+protected:
+    void                     preprocessStage(const VRect &clip) final;
+    void                     updateContent() final;
+    std::vector<VDrawable *> mDrawableList;
+    Group *                  mRoot{nullptr};
+};
+
+class NullLayer final : public Layer {
+public:
+    explicit NullLayer(model::Layer *layerData);
+
+protected:
+    void preprocessStage(const VRect &) final {}
+    void updateContent() final;
+};
+
+class ImageLayer final : public Layer {
+public:
+    explicit ImageLayer(model::Layer *layerData);
+    void         buildLayerNode() final;
+    DrawableList renderList() final;
+
+protected:
+    void preprocessStage(const VRect &clip) final;
+    void updateContent() final;
+
+private:
+    Drawable   mRenderNode;
+    VTexture   mTexture;
+    VPath      mPath;
+    VDrawable *mDrawableList{nullptr};  // to work with the Span api
+};
+
+class Object {
+public:
+    enum class Type : uchar { Unknown, Group, Shape, Paint, Trim };
+    virtual ~Object() = default;
+    Object &     operator=(Object &&) noexcept = delete;
+    virtual void update(int frameNo, const VMatrix &parentMatrix,
+                        float parentAlpha, const DirtyFlag &flag) = 0;
+    virtual void renderList(std::vector<VDrawable *> &) {}
+    virtual bool resolveKeyPath(LOTKeyPath &, uint, LOTVariant &)
+    {
+        return false;
+    }
+    virtual Object::Type type() const { return Object::Type::Unknown; }
+};
+
+class Shape;
+class Group : public Object {
+public:
+    Group() = default;
+    explicit Group(model::Group *data, VArenaAlloc *allocator);
+    void addChildren(model::Group *data, VArenaAlloc *allocator);
+    void update(int frameNo, const VMatrix &parentMatrix, float parentAlpha,
+                const DirtyFlag &flag) override;
+    void applyTrim();
+    void processTrimItems(std::vector<Shape *> &list);
+    void processPaintItems(std::vector<Shape *> &list);
+    void renderList(std::vector<VDrawable *> &list) override;
+    Object::Type   type() const final { return Object::Type::Group; }
+    const VMatrix &matrix() const { return mMatrix; }
+    const char *   name() const
+    {
+        static const char *TAG = "__";
+        return mModel.hasModel() ? mModel.name() : TAG;
+    }
+    bool resolveKeyPath(LOTKeyPath &keyPath, uint depth,
+                        LOTVariant &value) override;
+
+protected:
+    std::vector<Object *> mContents;
+    VMatrix               mMatrix;
+
+private:
+    model::Filter<model::Group> mModel;
+};
+
+class Shape : public Object {
+public:
+    Shape(bool staticPath) : mStaticPath(staticPath) {}
+    void update(int frameNo, const VMatrix &parentMatrix, float parentAlpha,
+                const DirtyFlag &flag) final;
+    Object::Type type() const final { return Object::Type::Shape; }
+    bool         dirty() const { return mDirtyPath; }
+    const VPath &localPath() const { return mTemp; }
+    void         finalPath(VPath &result);
+    void         updatePath(const VPath &path)
+    {
+        mTemp = path;
+        mDirtyPath = true;
+    }
+    bool   staticPath() const { return mStaticPath; }
+    void   setParent(Group *parent) { mParent = parent; }
+    Group *parent() const { return mParent; }
+
+protected:
+    virtual void updatePath(VPath &path, int frameNo) = 0;
+    virtual bool hasChanged(int prevFrame, int curFrame) = 0;
+
+private:
+    bool hasChanged(int frameNo)
+    {
+        int prevFrame = mFrameNo;
+        mFrameNo = frameNo;
+        if (prevFrame == -1) return true;
+        if (mStaticPath || (prevFrame == frameNo)) return false;
+        return hasChanged(prevFrame, frameNo);
+    }
+    Group *mParent{nullptr};
+    VPath  mLocalPath;
+    VPath  mTemp;
+    int    mFrameNo{-1};
+    bool   mDirtyPath{true};
+    bool   mStaticPath;
+};
+
+class Rect final : public Shape {
+public:
+    explicit Rect(model::Rect *data);
+
+protected:
+    void         updatePath(VPath &path, int frameNo) final;
+    model::Rect *mData{nullptr};
+
+    bool hasChanged(int prevFrame, int curFrame) final
+    {
+        return (mData->mPos.changed(prevFrame, curFrame) ||
+                mData->mSize.changed(prevFrame, curFrame) ||
+                mData->roundnessChanged(prevFrame, curFrame));
+    }
+};
+
+class Ellipse final : public Shape {
+public:
+    explicit Ellipse(model::Ellipse *data);
+
+private:
+    void            updatePath(VPath &path, int frameNo) final;
+    model::Ellipse *mData{nullptr};
+    bool            hasChanged(int prevFrame, int curFrame) final
+    {
+        return (mData->mPos.changed(prevFrame, curFrame) ||
+                mData->mSize.changed(prevFrame, curFrame));
+    }
+};
+
+class Path final : public Shape {
+public:
+    explicit Path(model::Path *data);
+
+private:
+    void         updatePath(VPath &path, int frameNo) final;
+    model::Path *mData{nullptr};
+    bool         hasChanged(int prevFrame, int curFrame) final
+    {
+        return mData->mShape.changed(prevFrame, curFrame);
+    }
+};
+
+class Polystar final : public Shape {
+public:
+    explicit Polystar(model::Polystar *data);
+
+private:
+    void             updatePath(VPath &path, int frameNo) final;
+    model::Polystar *mData{nullptr};
+
+    bool hasChanged(int prevFrame, int curFrame) final
+    {
+        return (mData->mPos.changed(prevFrame, curFrame) ||
+                mData->mPointCount.changed(prevFrame, curFrame) ||
+                mData->mInnerRadius.changed(prevFrame, curFrame) ||
+                mData->mOuterRadius.changed(prevFrame, curFrame) ||
+                mData->mInnerRoundness.changed(prevFrame, curFrame) ||
+                mData->mOuterRoundness.changed(prevFrame, curFrame) ||
+                mData->mRotation.changed(prevFrame, curFrame));
+    }
+};
+
+class Paint : public Object {
+public:
+    Paint(bool staticContent);
+    void addPathItems(std::vector<Shape *> &list, size_t startOffset);
+    void update(int frameNo, const VMatrix &parentMatrix, float parentAlpha,
+                const DirtyFlag &flag) override;
+    void renderList(std::vector<VDrawable *> &list) final;
+    Object::Type type() const final { return Object::Type::Paint; }
+
+protected:
+    virtual bool updateContent(int frameNo, const VMatrix &matrix,
+                               float alpha) = 0;
+
+private:
+    void updateRenderNode();
+
+protected:
+    std::vector<Shape *> mPathItems;
+    Drawable             mDrawable;
+    VPath                mPath;
+    DirtyFlag            mFlag;
+    bool                 mStaticContent;
+    bool                 mRenderNodeUpdate{true};
+    bool                 mContentToRender{true};
+};
+
+class Fill final : public Paint {
+public:
+    explicit Fill(model::Fill *data);
+
+protected:
+    bool updateContent(int frameNo, const VMatrix &matrix, float alpha) final;
+    bool resolveKeyPath(LOTKeyPath &keyPath, uint depth,
+                        LOTVariant &value) final;
+
+private:
+    model::Filter<model::Fill> mModel;
+};
+
+class GradientFill final : public Paint {
+public:
+    explicit GradientFill(model::GradientFill *data);
+
+protected:
+    bool updateContent(int frameNo, const VMatrix &matrix, float alpha) final;
+
+private:
+    model::GradientFill *      mData{nullptr};
+    std::unique_ptr<VGradient> mGradient;
+};
+
+class Stroke : public Paint {
+public:
+    explicit Stroke(model::Stroke *data);
+
+protected:
+    bool updateContent(int frameNo, const VMatrix &matrix, float alpha) final;
+    bool resolveKeyPath(LOTKeyPath &keyPath, uint depth,
+                        LOTVariant &value) final;
+
+private:
+    model::Filter<model::Stroke> mModel;
+};
+
+class GradientStroke final : public Paint {
+public:
+    explicit GradientStroke(model::GradientStroke *data);
+
+protected:
+    bool updateContent(int frameNo, const VMatrix &matrix, float alpha) final;
+
+private:
+    model::GradientStroke *    mData{nullptr};
+    std::unique_ptr<VGradient> mGradient;
+};
+
+class Trim final : public Object {
+public:
+    explicit Trim(model::Trim *data) : mData(data) {}
+    void update(int frameNo, const VMatrix &parentMatrix, float parentAlpha,
+                const DirtyFlag &flag) final;
+    Object::Type type() const final { return Object::Type::Trim; }
+    void         update();
+    void         addPathItems(std::vector<Shape *> &list, size_t startOffset);
+
+private:
+    bool pathDirty() const
+    {
+        for (auto &i : mPathItems) {
+            if (i->dirty()) return true;
+        }
+        return false;
+    }
+    struct Cache {
+        int                  mFrameNo{-1};
+        model::Trim::Segment mSegment{};
+    };
+    Cache                mCache;
+    std::vector<Shape *> mPathItems;
+    model::Trim *        mData{nullptr};
+    VPathMesure          mPathMesure;
+    bool                 mDirty{true};
+};
+
+class Repeater final : public Group {
+public:
+    explicit Repeater(model::Repeater *data, VArenaAlloc *allocator);
+    void update(int frameNo, const VMatrix &parentMatrix, float parentAlpha,
+                const DirtyFlag &flag) final;
+    void renderList(std::vector<VDrawable *> &list) final;
+
+private:
+    model::Repeater *mRepeaterData{nullptr};
+    bool             mHidden{false};
+    int              mCopies{0};
+};
+
+}  // namespace renderer
+
+}  // namespace internal
+
+}  // namespace rlottie
+
+#endif  // LOTTIEITEM_H

vendor/github.com/Benau/go_rlottie/lottie_lottieitem_capi.cpp

@@ -0,0 +1,339 @@
+/*
+ * Implements LottieItem functions needed
+ * to support renderTree() api.
+ * Moving all those implementation to its own
+ * file make clear separation as well easy of
+ * maintenance.
+ */
+
+#include "lottie_lottieitem.h"
+#include "vector_vdasher.h"
+
+using namespace rlottie::internal;
+
+renderer::CApiData::CApiData()
+{
+    mLayer.mMaskList.ptr = nullptr;
+    mLayer.mMaskList.size = 0;
+    mLayer.mLayerList.ptr = nullptr;
+    mLayer.mLayerList.size = 0;
+    mLayer.mNodeList.ptr = nullptr;
+    mLayer.mNodeList.size = 0;
+    mLayer.mMatte = MatteNone;
+    mLayer.mVisible = 0;
+    mLayer.mAlpha = 255;
+    mLayer.mClipPath.ptPtr = nullptr;
+    mLayer.mClipPath.elmPtr = nullptr;
+    mLayer.mClipPath.ptCount = 0;
+    mLayer.mClipPath.elmCount = 0;
+    mLayer.keypath = nullptr;
+}
+
+void renderer::Composition::buildRenderTree()
+{
+    mRootLayer->buildLayerNode();
+}
+
+const LOTLayerNode *renderer::Composition::renderTree() const
+{
+    return &mRootLayer->clayer();
+}
+
+void renderer::CompLayer::buildLayerNode()
+{
+    renderer::Layer::buildLayerNode();
+    if (mClipper) {
+        const auto &elm = mClipper->mPath.elements();
+        const auto &pts = mClipper->mPath.points();
+        auto        ptPtr = reinterpret_cast<const float *>(pts.data());
+        auto        elmPtr = reinterpret_cast<const char *>(elm.data());
+        clayer().mClipPath.ptPtr = ptPtr;
+        clayer().mClipPath.elmPtr = elmPtr;
+        clayer().mClipPath.ptCount = 2 * pts.size();
+        clayer().mClipPath.elmCount = elm.size();
+    }
+    if (mLayers.size() != clayers().size()) {
+        for (const auto &layer : mLayers) {
+            layer->buildLayerNode();
+            clayers().push_back(&layer->clayer());
+        }
+        clayer().mLayerList.ptr = clayers().data();
+        clayer().mLayerList.size = clayers().size();
+    } else {
+        for (const auto &layer : mLayers) {
+            layer->buildLayerNode();
+        }
+    }
+}
+
+void renderer::ShapeLayer::buildLayerNode()
+{
+    renderer::Layer::buildLayerNode();
+
+    auto renderlist = renderList();
+
+    cnodes().clear();
+    for (auto &i : renderlist) {
+        auto lotDrawable = static_cast<renderer::Drawable *>(i);
+        lotDrawable->sync();
+        cnodes().push_back(lotDrawable->mCNode.get());
+    }
+    clayer().mNodeList.ptr = cnodes().data();
+    clayer().mNodeList.size = cnodes().size();
+}
+
+void renderer::Layer::buildLayerNode()
+{
+    if (!mCApiData) {
+        mCApiData = std::make_unique<renderer::CApiData>();
+        clayer().keypath = name();
+    }
+    if (complexContent()) clayer().mAlpha = uchar(combinedAlpha() * 255.f);
+    clayer().mVisible = visible();
+    // update matte
+    if (hasMatte()) {
+        switch (mLayerData->mMatteType) {
+        case model::MatteType::Alpha:
+            clayer().mMatte = MatteAlpha;
+            break;
+        case model::MatteType::AlphaInv:
+            clayer().mMatte = MatteAlphaInv;
+            break;
+        case model::MatteType::Luma:
+            clayer().mMatte = MatteLuma;
+            break;
+        case model::MatteType::LumaInv:
+            clayer().mMatte = MatteLumaInv;
+            break;
+        default:
+            clayer().mMatte = MatteNone;
+            break;
+        }
+    }
+    if (mLayerMask) {
+        cmasks().clear();
+        cmasks().resize(mLayerMask->mMasks.size());
+        size_t i = 0;
+        for (const auto &mask : mLayerMask->mMasks) {
+            auto &      cNode = cmasks()[i++];
+            const auto &elm = mask.mFinalPath.elements();
+            const auto &pts = mask.mFinalPath.points();
+            auto        ptPtr = reinterpret_cast<const float *>(pts.data());
+            auto        elmPtr = reinterpret_cast<const char *>(elm.data());
+            cNode.mPath.ptPtr = ptPtr;
+            cNode.mPath.ptCount = 2 * pts.size();
+            cNode.mPath.elmPtr = elmPtr;
+            cNode.mPath.elmCount = elm.size();
+            cNode.mAlpha = uchar(mask.mCombinedAlpha * 255.0f);
+            switch (mask.maskMode()) {
+            case model::Mask::Mode::Add:
+                cNode.mMode = MaskAdd;
+                break;
+            case model::Mask::Mode::Substarct:
+                cNode.mMode = MaskSubstract;
+                break;
+            case model::Mask::Mode::Intersect:
+                cNode.mMode = MaskIntersect;
+                break;
+            case model::Mask::Mode::Difference:
+                cNode.mMode = MaskDifference;
+                break;
+            default:
+                cNode.mMode = MaskAdd;
+                break;
+            }
+        }
+        clayer().mMaskList.ptr = cmasks().data();
+        clayer().mMaskList.size = cmasks().size();
+    }
+}
+
+void renderer::SolidLayer::buildLayerNode()
+{
+    renderer::Layer::buildLayerNode();
+
+    auto renderlist = renderList();
+
+    cnodes().clear();
+    for (auto &i : renderlist) {
+        auto lotDrawable = static_cast<renderer::Drawable *>(i);
+        lotDrawable->sync();
+        cnodes().push_back(lotDrawable->mCNode.get());
+    }
+    clayer().mNodeList.ptr = cnodes().data();
+    clayer().mNodeList.size = cnodes().size();
+}
+
+void renderer::ImageLayer::buildLayerNode()
+{
+    renderer::Layer::buildLayerNode();
+
+    auto renderlist = renderList();
+
+    cnodes().clear();
+    for (auto &i : renderlist) {
+        auto lotDrawable = static_cast<renderer::Drawable *>(i);
+        lotDrawable->sync();
+
+        lotDrawable->mCNode->mImageInfo.data =
+            lotDrawable->mBrush.mTexture->mBitmap.data();
+        lotDrawable->mCNode->mImageInfo.width =
+            int(lotDrawable->mBrush.mTexture->mBitmap.width());
+        lotDrawable->mCNode->mImageInfo.height =
+            int(lotDrawable->mBrush.mTexture->mBitmap.height());
+
+        lotDrawable->mCNode->mImageInfo.mMatrix.m11 = combinedMatrix().m_11();
+        lotDrawable->mCNode->mImageInfo.mMatrix.m12 = combinedMatrix().m_12();
+        lotDrawable->mCNode->mImageInfo.mMatrix.m13 = combinedMatrix().m_13();
+
+        lotDrawable->mCNode->mImageInfo.mMatrix.m21 = combinedMatrix().m_21();
+        lotDrawable->mCNode->mImageInfo.mMatrix.m22 = combinedMatrix().m_22();
+        lotDrawable->mCNode->mImageInfo.mMatrix.m23 = combinedMatrix().m_23();
+
+        lotDrawable->mCNode->mImageInfo.mMatrix.m31 = combinedMatrix().m_tx();
+        lotDrawable->mCNode->mImageInfo.mMatrix.m32 = combinedMatrix().m_ty();
+        lotDrawable->mCNode->mImageInfo.mMatrix.m33 = combinedMatrix().m_33();
+
+        // Alpha calculation already combined.
+        lotDrawable->mCNode->mImageInfo.mAlpha =
+            uchar(lotDrawable->mBrush.mTexture->mAlpha);
+
+        cnodes().push_back(lotDrawable->mCNode.get());
+    }
+    clayer().mNodeList.ptr = cnodes().data();
+    clayer().mNodeList.size = cnodes().size();
+}
+
+static void updateGStops(LOTNode *n, const VGradient *grad)
+{
+    if (grad->mStops.size() != n->mGradient.stopCount) {
+        if (n->mGradient.stopCount) free(n->mGradient.stopPtr);
+        n->mGradient.stopCount = grad->mStops.size();
+        n->mGradient.stopPtr = (LOTGradientStop *)malloc(
+            n->mGradient.stopCount * sizeof(LOTGradientStop));
+    }
+
+    LOTGradientStop *ptr = n->mGradient.stopPtr;
+    for (const auto &i : grad->mStops) {
+        ptr->pos = i.first;
+        ptr->a = uchar(i.second.alpha() * grad->alpha());
+        ptr->r = i.second.red();
+        ptr->g = i.second.green();
+        ptr->b = i.second.blue();
+        ptr++;
+    }
+}
+
+void renderer::Drawable::sync()
+{
+    if (!mCNode) {
+        mCNode = std::make_unique<LOTNode>();
+        mCNode->mGradient.stopPtr = nullptr;
+        mCNode->mGradient.stopCount = 0;
+    }
+
+    mCNode->mFlag = ChangeFlagNone;
+    if (mFlag & DirtyState::None) return;
+
+    if (mFlag & DirtyState::Path) {
+        applyDashOp();
+        const std::vector<VPath::Element> &elm = mPath.elements();
+        const std::vector<VPointF> &       pts = mPath.points();
+        const float *ptPtr = reinterpret_cast<const float *>(pts.data());
+        const char * elmPtr = reinterpret_cast<const char *>(elm.data());
+        mCNode->mPath.elmPtr = elmPtr;
+        mCNode->mPath.elmCount = elm.size();
+        mCNode->mPath.ptPtr = ptPtr;
+        mCNode->mPath.ptCount = 2 * pts.size();
+        mCNode->mFlag |= ChangeFlagPath;
+        mCNode->keypath = name();
+    }
+
+    if (mStrokeInfo) {
+        mCNode->mStroke.width = mStrokeInfo->width;
+        mCNode->mStroke.miterLimit = mStrokeInfo->miterLimit;
+        mCNode->mStroke.enable = 1;
+
+        switch (mStrokeInfo->cap) {
+        case CapStyle::Flat:
+            mCNode->mStroke.cap = LOTCapStyle::CapFlat;
+            break;
+        case CapStyle::Square:
+            mCNode->mStroke.cap = LOTCapStyle::CapSquare;
+            break;
+        case CapStyle::Round:
+            mCNode->mStroke.cap = LOTCapStyle::CapRound;
+            break;
+        }
+
+        switch (mStrokeInfo->join) {
+        case JoinStyle::Miter:
+            mCNode->mStroke.join = LOTJoinStyle::JoinMiter;
+            break;
+        case JoinStyle::Bevel:
+            mCNode->mStroke.join = LOTJoinStyle::JoinBevel;
+            break;
+        case JoinStyle::Round:
+            mCNode->mStroke.join = LOTJoinStyle::JoinRound;
+            break;
+        default:
+            mCNode->mStroke.join = LOTJoinStyle::JoinMiter;
+            break;
+        }
+    } else {
+        mCNode->mStroke.enable = 0;
+    }
+
+    switch (mFillRule) {
+    case FillRule::EvenOdd:
+        mCNode->mFillRule = LOTFillRule::FillEvenOdd;
+        break;
+    default:
+        mCNode->mFillRule = LOTFillRule::FillWinding;
+        break;
+    }
+
+    switch (mBrush.type()) {
+    case VBrush::Type::Solid:
+        mCNode->mBrushType = LOTBrushType::BrushSolid;
+        mCNode->mColor.r = mBrush.mColor.r;
+        mCNode->mColor.g = mBrush.mColor.g;
+        mCNode->mColor.b = mBrush.mColor.b;
+        mCNode->mColor.a = mBrush.mColor.a;
+        break;
+    case VBrush::Type::LinearGradient: {
+        mCNode->mBrushType = LOTBrushType::BrushGradient;
+        mCNode->mGradient.type = LOTGradientType::GradientLinear;
+        VPointF s = mBrush.mGradient->mMatrix.map(
+            {mBrush.mGradient->linear.x1, mBrush.mGradient->linear.y1});
+        VPointF e = mBrush.mGradient->mMatrix.map(
+            {mBrush.mGradient->linear.x2, mBrush.mGradient->linear.y2});
+        mCNode->mGradient.start.x = s.x();
+        mCNode->mGradient.start.y = s.y();
+        mCNode->mGradient.end.x = e.x();
+        mCNode->mGradient.end.y = e.y();
+        updateGStops(mCNode.get(), mBrush.mGradient);
+        break;
+    }
+    case VBrush::Type::RadialGradient: {
+        mCNode->mBrushType = LOTBrushType::BrushGradient;
+        mCNode->mGradient.type = LOTGradientType::GradientRadial;
+        VPointF c = mBrush.mGradient->mMatrix.map(
+            {mBrush.mGradient->radial.cx, mBrush.mGradient->radial.cy});
+        VPointF f = mBrush.mGradient->mMatrix.map(
+            {mBrush.mGradient->radial.fx, mBrush.mGradient->radial.fy});
+        mCNode->mGradient.center.x = c.x();
+        mCNode->mGradient.center.y = c.y();
+        mCNode->mGradient.focal.x = f.x();
+        mCNode->mGradient.focal.y = f.y();
+
+        float scale = mBrush.mGradient->mMatrix.scale();
+        mCNode->mGradient.cradius = mBrush.mGradient->radial.cradius * scale;
+        mCNode->mGradient.fradius = mBrush.mGradient->radial.fradius * scale;
+        updateGStops(mCNode.get(), mBrush.mGradient);
+        break;
+    }
+    default:
+        break;
+    }
+}

vendor/github.com/Benau/go_rlottie/lottie_lottiekeypath.cpp

@@ -0,0 +1,86 @@
+#include "lottie_lottiekeypath.h"
+
+#include <sstream>
+
+LOTKeyPath::LOTKeyPath(const std::string &keyPath)
+{
+    std::stringstream ss(keyPath);
+    std::string       item;
+
+    while (getline(ss, item, '.')) {
+        mKeys.push_back(item);
+    }
+}
+
+bool LOTKeyPath::matches(const std::string &key, uint depth)
+{
+    if (skip(key)) {
+        // This is an object we programatically create.
+        return true;
+    }
+    if (depth > size()) {
+        return false;
+    }
+    if ((mKeys[depth] == key) || (mKeys[depth] == "*") ||
+        (mKeys[depth] == "**")) {
+        return true;
+    }
+    return false;
+}
+
+uint LOTKeyPath::nextDepth(const std::string key, uint depth)
+{
+    if (skip(key)) {
+        // If it's a container then we added programatically and it isn't a part
+        // of the keypath.
+        return depth;
+    }
+    if (mKeys[depth] != "**") {
+        // If it's not a globstar then it is part of the keypath.
+        return depth + 1;
+    }
+    if (depth == size()) {
+        // The last key is a globstar.
+        return depth;
+    }
+    if (mKeys[depth + 1] == key) {
+        // We are a globstar and the next key is our current key so consume
+        // both.
+        return depth + 2;
+    }
+    return depth;
+}
+
+bool LOTKeyPath::fullyResolvesTo(const std::string key, uint depth)
+{
+    if (depth > mKeys.size()) {
+        return false;
+    }
+
+    bool isLastDepth = (depth == size());
+
+    if (!isGlobstar(depth)) {
+        bool matches = (mKeys[depth] == key) || isGlob(depth);
+        return (isLastDepth || (depth == size() - 1 && endsWithGlobstar())) &&
+               matches;
+    }
+
+    bool isGlobstarButNextKeyMatches = !isLastDepth && mKeys[depth + 1] == key;
+    if (isGlobstarButNextKeyMatches) {
+        return depth == size() - 1 ||
+               (depth == size() - 2 && endsWithGlobstar());
+    }
+
+    if (isLastDepth) {
+        return true;
+    }
+
+    if (depth + 1 < size()) {
+        // We are a globstar but there is more than 1 key after the globstar we
+        // we can't fully match.
+        return false;
+    }
+    // Return whether the next key (which we now know is the last one) is the
+    // same as the current key.
+    return mKeys[depth + 1] == key;
+}

vendor/github.com/Benau/go_rlottie/lottie_lottiekeypath.h

@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2020 Samsung Electronics Co., Ltd. All rights reserved.
+
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef LOTTIEKEYPATH_H
+#define LOTTIEKEYPATH_H
+
+#include <string>
+#include <vector>
+#include "vector_vglobal.h"
+
+class LOTKeyPath {
+public:
+    LOTKeyPath(const std::string &keyPath);
+    bool matches(const std::string &key, uint depth);
+    uint nextDepth(const std::string key, uint depth);
+    bool fullyResolvesTo(const std::string key, uint depth);
+
+    bool propagate(const std::string key, uint depth)
+    {
+        return skip(key) ? true : (depth < size()) || (mKeys[depth] == "**");
+    }
+    bool skip(const std::string &key) const { return key == "__"; }
+
+private:
+    bool   isGlobstar(uint depth) const { return mKeys[depth] == "**"; }
+    bool   isGlob(uint depth) const { return mKeys[depth] == "*"; }
+    bool   endsWithGlobstar() const { return mKeys.back() == "**"; }
+    size_t size() const { return mKeys.size() - 1; }
+
+private:
+    std::vector<std::string> mKeys;
+};
+
+#endif  // LOTTIEKEYPATH_H

vendor/github.com/Benau/go_rlottie/lottie_lottieloader.cpp

@@ -0,0 +1,169 @@
+/*
+ * Copyright (c) 2020 Samsung Electronics Co., Ltd. All rights reserved.
+
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <cstring>
+#include <fstream>
+#include <sstream>
+
+#include "lottie_lottiemodel.h"
+
+using namespace rlottie::internal;
+
+#ifdef LOTTIE_CACHE_SUPPORT
+
+#include <mutex>
+#include <unordered_map>
+
+class ModelCache {
+public:
+    static ModelCache &instance()
+    {
+        static ModelCache singleton;
+        return singleton;
+    }
+    std::shared_ptr<model::Composition> find(const std::string &key)
+    {
+        std::lock_guard<std::mutex> guard(mMutex);
+
+        if (!mcacheSize) return nullptr;
+
+        auto search = mHash.find(key);
+
+        return (search != mHash.end()) ? search->second : nullptr;
+    }
+    void add(const std::string &key, std::shared_ptr<model::Composition> value)
+    {
+        std::lock_guard<std::mutex> guard(mMutex);
+
+        if (!mcacheSize) return;
+
+        //@TODO just remove the 1st element
+        // not the best of LRU logic
+        if (mcacheSize == mHash.size()) mHash.erase(mHash.cbegin());
+
+        mHash[key] = std::move(value);
+    }
+
+    void configureCacheSize(size_t cacheSize)
+    {
+        std::lock_guard<std::mutex> guard(mMutex);
+        mcacheSize = cacheSize;
+
+        if (!mcacheSize) mHash.clear();
+    }
+
+private:
+    ModelCache() = default;
+
+    std::unordered_map<std::string, std::shared_ptr<model::Composition>> mHash;
+    std::mutex                                                           mMutex;
+    size_t mcacheSize{10};
+};
+
+#else
+
+class ModelCache {
+public:
+    static ModelCache &instance()
+    {
+        static ModelCache singleton;
+        return singleton;
+    }
+    std::shared_ptr<model::Composition> find(const std::string &)
+    {
+        return nullptr;
+    }
+    void add(const std::string &, std::shared_ptr<model::Composition>) {}
+    void configureCacheSize(size_t) {}
+};
+
+#endif
+
+static std::string dirname(const std::string &path)
+{
+    const char *ptr = strrchr(path.c_str(), '/');
+#ifdef _WIN32
+    if (ptr) ptr = strrchr(ptr + 1, '\\');
+#endif
+    int len = int(ptr + 1 - path.c_str());  // +1 to include '/'
+    return std::string(path, 0, len);
+}
+
+void model::configureModelCacheSize(size_t cacheSize)
+{
+    ModelCache::instance().configureCacheSize(cacheSize);
+}
+
+std::shared_ptr<model::Composition> model::loadFromFile(const std::string &path,
+                                                        bool cachePolicy)
+{
+    if (cachePolicy) {
+        auto obj = ModelCache::instance().find(path);
+        if (obj) return obj;
+    }
+
+    std::ifstream f;
+    f.open(path);
+
+    if (!f.is_open()) {
+        vCritical << "failed to open file = " << path.c_str();
+        return {};
+    } else {
+        std::string content;
+
+        std::getline(f, content, '\0');
+        f.close();
+
+        if (content.empty()) return {};
+
+        auto obj = internal::model::parse(const_cast<char *>(content.c_str()),
+                                          dirname(path));
+
+        if (obj && cachePolicy) ModelCache::instance().add(path, obj);
+
+        return obj;
+    }
+}
+
+std::shared_ptr<model::Composition> model::loadFromData(
+    std::string jsonData, const std::string &key, std::string resourcePath,
+    bool cachePolicy)
+{
+    if (cachePolicy) {
+        auto obj = ModelCache::instance().find(key);
+        if (obj) return obj;
+    }
+
+    auto obj = internal::model::parse(const_cast<char *>(jsonData.c_str()),
+                                      std::move(resourcePath));
+
+    if (obj && cachePolicy) ModelCache::instance().add(key, obj);
+
+    return obj;
+}
+
+std::shared_ptr<model::Composition> model::loadFromData(
+    std::string jsonData, std::string resourcePath, model::ColorFilter filter)
+{
+    return internal::model::parse(const_cast<char *>(jsonData.c_str()),
+                                  std::move(resourcePath), std::move(filter));
+}

vendor/github.com/Benau/go_rlottie/lottie_lottiemodel.cpp

@@ -0,0 +1,390 @@
+/*
+ * Copyright (c) 2020 Samsung Electronics Co., Ltd. All rights reserved.
+
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "lottie_lottiemodel.h"
+#include <cassert>
+#include <iterator>
+#include <stack>
+#include "vector_vimageloader.h"
+#include "vector_vline.h"
+
+using namespace rlottie::internal;
+
+/*
+ * We process the iterator objects in the children list
+ * by iterating from back to front. when we find a repeater object
+ * we remove the objects from satrt till repeater object and then place
+ * under a new shape group object which we add it as children to the repeater
+ * object.
+ * Then we visit the childrens of the newly created shape group object to
+ * process the remaining repeater object(when children list contains more than
+ * one repeater).
+ *
+ */
+class LottieRepeaterProcesser {
+public:
+    void visitChildren(model::Group *obj)
+    {
+        for (auto i = obj->mChildren.rbegin(); i != obj->mChildren.rend();
+             ++i) {
+            auto child = (*i);
+            if (child->type() == model::Object::Type::Repeater) {
+                model::Repeater *repeater =
+                    static_cast<model::Repeater *>(child);
+                // check if this repeater is already processed
+                // can happen if the layer is an asset and referenced by
+                // multiple layer.
+                if (repeater->processed()) continue;
+
+                repeater->markProcessed();
+
+                auto content = repeater->content();
+                // 1. increment the reverse iterator to point to the
+                //   object before the repeater
+                ++i;
+                // 2. move all the children till repater to the group
+                std::move(obj->mChildren.begin(), i.base(),
+                          back_inserter(content->mChildren));
+                // 3. erase the objects from the original children list
+                obj->mChildren.erase(obj->mChildren.begin(), i.base());
+
+                // 5. visit newly created group to process remaining repeater
+                // object.
+                visitChildren(content);
+                // 6. exit the loop as the current iterators are invalid
+                break;
+            }
+            visit(child);
+        }
+    }
+
+    void visit(model::Object *obj)
+    {
+        switch (obj->type()) {
+        case model::Object::Type::Group:
+        case model::Object::Type::Layer: {
+            visitChildren(static_cast<model::Group *>(obj));
+            break;
+        }
+        default:
+            break;
+        }
+    }
+};
+
+class LottieUpdateStatVisitor {
+    model::Composition::Stats *stat;
+
+public:
+    explicit LottieUpdateStatVisitor(model::Composition::Stats *s) : stat(s) {}
+    void visitChildren(model::Group *obj)
+    {
+        for (const auto &child : obj->mChildren) {
+            if (child) visit(child);
+        }
+    }
+    void visitLayer(model::Layer *layer)
+    {
+        switch (layer->mLayerType) {
+        case model::Layer::Type::Precomp:
+            stat->precompLayerCount++;
+            break;
+        case model::Layer::Type::Null:
+            stat->nullLayerCount++;
+            break;
+        case model::Layer::Type::Shape:
+            stat->shapeLayerCount++;
+            break;
+        case model::Layer::Type::Solid:
+            stat->solidLayerCount++;
+            break;
+        case model::Layer::Type::Image:
+            stat->imageLayerCount++;
+            break;
+        default:
+            break;
+        }
+        visitChildren(layer);
+    }
+    void visit(model::Object *obj)
+    {
+        switch (obj->type()) {
+        case model::Object::Type::Layer: {
+            visitLayer(static_cast<model::Layer *>(obj));
+            break;
+        }
+        case model::Object::Type::Repeater: {
+            visitChildren(static_cast<model::Repeater *>(obj)->content());
+            break;
+        }
+        case model::Object::Type::Group: {
+            visitChildren(static_cast<model::Group *>(obj));
+            break;
+        }
+        default:
+            break;
+        }
+    }
+};
+
+void model::Composition::processRepeaterObjects()
+{
+    LottieRepeaterProcesser visitor;
+    visitor.visit(mRootLayer);
+}
+
+void model::Composition::updateStats()
+{
+    LottieUpdateStatVisitor visitor(&mStats);
+    visitor.visit(mRootLayer);
+}
+
+VMatrix model::Repeater::Transform::matrix(int frameNo, float multiplier) const
+{
+    VPointF scale = mScale.value(frameNo) / 100.f;
+    scale.setX(std::pow(scale.x(), multiplier));
+    scale.setY(std::pow(scale.y(), multiplier));
+    VMatrix m;
+    m.translate(mPosition.value(frameNo) * multiplier)
+        .translate(mAnchor.value(frameNo))
+        .scale(scale)
+        .rotate(mRotation.value(frameNo) * multiplier)
+        .translate(-mAnchor.value(frameNo));
+
+    return m;
+}
+
+VMatrix model::Transform::Data::matrix(int frameNo, bool autoOrient) const
+{
+    VMatrix m;
+    VPointF position;
+    if (mExtra && mExtra->mSeparate) {
+        position.setX(mExtra->mSeparateX.value(frameNo));
+        position.setY(mExtra->mSeparateY.value(frameNo));
+    } else {
+        position = mPosition.value(frameNo);
+    }
+
+    float angle = autoOrient ? mPosition.angle(frameNo) : 0;
+    if (mExtra && mExtra->m3DData) {
+        m.translate(position)
+            .rotate(mExtra->m3DRz.value(frameNo) + angle)
+            .rotate(mExtra->m3DRy.value(frameNo), VMatrix::Axis::Y)
+            .rotate(mExtra->m3DRx.value(frameNo), VMatrix::Axis::X)
+            .scale(mScale.value(frameNo) / 100.f)
+            .translate(-mAnchor.value(frameNo));
+    } else {
+        m.translate(position)
+            .rotate(mRotation.value(frameNo) + angle)
+            .scale(mScale.value(frameNo) / 100.f)
+            .translate(-mAnchor.value(frameNo));
+    }
+    return m;
+}
+
+void model::Dash::getDashInfo(int frameNo, std::vector<float> &result) const
+{
+    result.clear();
+
+    if (mData.size() <= 1) return;
+
+    if (result.capacity() < mData.size()) result.reserve(mData.size() + 1);
+
+    for (const auto &elm : mData) result.push_back(elm.value(frameNo));
+
+    // if the size is even then we are missing last
+    // gap information which is same as the last dash value
+    // copy it from the last dash value.
+    // NOTE: last value is the offset and last-1 is the last dash value.
+    auto size = result.size();
+    if ((size % 2) == 0) {
+        // copy offset value to end.
+        result.push_back(result.back());
+        // copy dash value to gap.
+        result[size - 1] = result[size - 2];
+    }
+}
+
+/**
+ * Both the color stops and opacity stops are in the same array.
+ * There are {@link #colorPoints} colors sequentially as:
+ * [
+ *     ...,
+ *     position,
+ *     red,
+ *     green,
+ *     blue,
+ *     ...
+ * ]
+ *
+ * The remainder of the array is the opacity stops sequentially as:
+ * [
+ *     ...,
+ *     position,
+ *     opacity,
+ *     ...
+ * ]
+ */
+void model::Gradient::populate(VGradientStops &stops, int frameNo)
+{
+    model::Gradient::Data gradData = mGradient.value(frameNo);
+    auto                  size = gradData.mGradient.size();
+    float *               ptr = gradData.mGradient.data();
+    int                   colorPoints = mColorPoints;
+    if (colorPoints == -1) {  // for legacy bodymovin (ref: lottie-android)
+        colorPoints = int(size / 4);
+    }
+    auto   opacityArraySize = size - colorPoints * 4;
+    float *opacityPtr = ptr + (colorPoints * 4);
+    stops.clear();
+    size_t j = 0;
+    for (int i = 0; i < colorPoints; i++) {
+        float        colorStop = ptr[0];
+        model::Color color = model::Color(ptr[1], ptr[2], ptr[3]);
+        if (opacityArraySize) {
+            if (j == opacityArraySize) {
+                // already reached the end
+                float stop1 = opacityPtr[j - 4];
+                float op1 = opacityPtr[j - 3];
+                float stop2 = opacityPtr[j - 2];
+                float op2 = opacityPtr[j - 1];
+                if (colorStop > stop2) {
+                    stops.push_back(
+                        std::make_pair(colorStop, color.toColor(op2)));
+                } else {
+                    float progress = (colorStop - stop1) / (stop2 - stop1);
+                    float opacity = op1 + progress * (op2 - op1);
+                    stops.push_back(
+                        std::make_pair(colorStop, color.toColor(opacity)));
+                }
+                continue;
+            }
+            for (; j < opacityArraySize; j += 2) {
+                float opacityStop = opacityPtr[j];
+                if (opacityStop < colorStop) {
+                    // add a color using opacity stop
+                    stops.push_back(std::make_pair(
+                        opacityStop, color.toColor(opacityPtr[j + 1])));
+                    continue;
+                }
+                // add a color using color stop
+                if (j == 0) {
+                    stops.push_back(std::make_pair(
+                        colorStop, color.toColor(opacityPtr[j + 1])));
+                } else {
+                    float progress = (colorStop - opacityPtr[j - 2]) /
+                                     (opacityPtr[j] - opacityPtr[j - 2]);
+                    float opacity =
+                        opacityPtr[j - 1] +
+                        progress * (opacityPtr[j + 1] - opacityPtr[j - 1]);
+                    stops.push_back(
+                        std::make_pair(colorStop, color.toColor(opacity)));
+                }
+                j += 2;
+                break;
+            }
+        } else {
+            stops.push_back(std::make_pair(colorStop, color.toColor()));
+        }
+        ptr += 4;
+    }
+}
+
+void model::Gradient::update(std::unique_ptr<VGradient> &grad, int frameNo)
+{
+    bool init = false;
+    if (!grad) {
+        if (mGradientType == 1)
+            grad = std::make_unique<VGradient>(VGradient::Type::Linear);
+        else
+            grad = std::make_unique<VGradient>(VGradient::Type::Radial);
+        grad->mSpread = VGradient::Spread::Pad;
+        init = true;
+    }
+
+    if (!mGradient.isStatic() || init) {
+        populate(grad->mStops, frameNo);
+    }
+
+    if (mGradientType == 1) {  // linear gradient
+        VPointF start = mStartPoint.value(frameNo);
+        VPointF end = mEndPoint.value(frameNo);
+        grad->linear.x1 = start.x();
+        grad->linear.y1 = start.y();
+        grad->linear.x2 = end.x();
+        grad->linear.y2 = end.y();
+    } else {  // radial gradient
+        VPointF start = mStartPoint.value(frameNo);
+        VPointF end = mEndPoint.value(frameNo);
+        grad->radial.cx = start.x();
+        grad->radial.cy = start.y();
+        grad->radial.cradius =
+            VLine::length(start.x(), start.y(), end.x(), end.y());
+        /*
+         * Focal point is the point lives in highlight length distance from
+         * center along the line (start, end)  and rotated by highlight angle.
+         * below calculation first finds the quadrant(angle) on which the point
+         * lives by applying inverse slope formula then adds the rotation angle
+         * to find the final angle. then point is retrived using circle equation
+         * of center, angle and distance.
+         */
+        float progress = mHighlightLength.value(frameNo) / 100.0f;
+        if (vCompare(progress, 1.0f)) progress = 0.99f;
+        float                  startAngle = VLine(start, end).angle();
+        float                  highlightAngle = mHighlightAngle.value(frameNo);
+        static constexpr float K_PI = 3.1415926f;
+        float angle = (startAngle + highlightAngle) * (K_PI / 180.0f);
+        grad->radial.fx =
+            grad->radial.cx + std::cos(angle) * progress * grad->radial.cradius;
+        grad->radial.fy =
+            grad->radial.cy + std::sin(angle) * progress * grad->radial.cradius;
+        // Lottie dosen't have any focal radius concept.
+        grad->radial.fradius = 0;
+    }
+}
+
+void model::Asset::loadImageData(std::string data)
+{
+    if (!data.empty())
+        mBitmap = VImageLoader::instance().load(data.c_str(), data.length());
+}
+
+void model::Asset::loadImagePath(std::string path)
+{
+    if (!path.empty()) mBitmap = VImageLoader::instance().load(path.c_str());
+}
+
+std::vector<LayerInfo> model::Composition::layerInfoList() const
+{
+    if (!mRootLayer || mRootLayer->mChildren.empty()) return {};
+
+    std::vector<LayerInfo> result;
+
+    result.reserve(mRootLayer->mChildren.size());
+
+    for (auto it : mRootLayer->mChildren) {
+        auto layer = static_cast<model::Layer *>(it);
+        result.emplace_back(layer->name(), layer->mInFrame, layer->mOutFrame);
+    }
+
+    return result;
+}

vendor/github.com/Benau/go_rlottie/lottie_rapidjson_allocators.h

@@ -0,0 +1,284 @@
+// Tencent is pleased to support the open source community by making RapidJSON available.
+// 
+// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved.
+//
+// Licensed under the MIT License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// http://opensource.org/licenses/MIT
+//
+// Unless required by applicable law or agreed to in writing, software distributed 
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the 
+// specific language governing permissions and limitations under the License.
+
+#ifndef RAPIDJSON_ALLOCATORS_H_
+#define RAPIDJSON_ALLOCATORS_H_
+
+#include "lottie_rapidjson_rapidjson.h"
+
+RAPIDJSON_NAMESPACE_BEGIN
+
+///////////////////////////////////////////////////////////////////////////////
+// Allocator
+
+/*! \class rapidjson::Allocator
+    \brief Concept for allocating, resizing and freeing memory block.
+    
+    Note that Malloc() and Realloc() are non-static but Free() is static.
+    
+    So if an allocator need to support Free(), it needs to put its pointer in 
+    the header of memory block.
+
+\code
+concept Allocator {
+    static const bool kNeedFree;    //!< Whether this allocator needs to call Free().
+
+    // Allocate a memory block.
+    // \param size of the memory block in bytes.
+    // \returns pointer to the memory block.
+    void* Malloc(size_t size);
+
+    // Resize a memory block.
+    // \param originalPtr The pointer to current memory block. Null pointer is permitted.
+    // \param originalSize The current size in bytes. (Design issue: since some allocator may not book-keep this, explicitly pass to it can save memory.)
+    // \param newSize the new size in bytes.
+    void* Realloc(void* originalPtr, size_t originalSize, size_t newSize);
+
+    // Free a memory block.
+    // \param pointer to the memory block. Null pointer is permitted.
+    static void Free(void *ptr);
+};
+\endcode
+*/
+
+
+/*! \def RAPIDJSON_ALLOCATOR_DEFAULT_CHUNK_CAPACITY
+    \ingroup RAPIDJSON_CONFIG
+    \brief User-defined kDefaultChunkCapacity definition.
+
+    User can define this as any \c size that is a power of 2.
+*/
+
+#ifndef RAPIDJSON_ALLOCATOR_DEFAULT_CHUNK_CAPACITY
+#define RAPIDJSON_ALLOCATOR_DEFAULT_CHUNK_CAPACITY (64 * 1024)
+#endif
+
+
+///////////////////////////////////////////////////////////////////////////////
+// CrtAllocator
+
+//! C-runtime library allocator.
+/*! This class is just wrapper for standard C library memory routines.
+    \note implements Allocator concept
+*/
+class CrtAllocator {
+public:
+    static const bool kNeedFree = true;
+    void* Malloc(size_t size) { 
+        if (size) //  behavior of malloc(0) is implementation defined.
+            return RAPIDJSON_MALLOC(size);
+        else
+            return NULL; // standardize to returning NULL.
+    }
+    void* Realloc(void* originalPtr, size_t originalSize, size_t newSize) {
+        (void)originalSize;
+        if (newSize == 0) {
+            RAPIDJSON_FREE(originalPtr);
+            return NULL;
+        }
+        return RAPIDJSON_REALLOC(originalPtr, newSize);
+    }
+    static void Free(void *ptr) { RAPIDJSON_FREE(ptr); }
+};
+
+///////////////////////////////////////////////////////////////////////////////
+// MemoryPoolAllocator
+
+//! Default memory allocator used by the parser and DOM.
+/*! This allocator allocate memory blocks from pre-allocated memory chunks. 
+
+    It does not free memory blocks. And Realloc() only allocate new memory.
+
+    The memory chunks are allocated by BaseAllocator, which is CrtAllocator by default.
+
+    User may also supply a buffer as the first chunk.
+
+    If the user-buffer is full then additional chunks are allocated by BaseAllocator.
+
+    The user-buffer is not deallocated by this allocator.
+
+    \tparam BaseAllocator the allocator type for allocating memory chunks. Default is CrtAllocator.
+    \note implements Allocator concept
+*/
+template <typename BaseAllocator = CrtAllocator>
+class MemoryPoolAllocator {
+public:
+    static const bool kNeedFree = false;    //!< Tell users that no need to call Free() with this allocator. (concept Allocator)
+
+    //! Constructor with chunkSize.
+    /*! \param chunkSize The size of memory chunk. The default is kDefaultChunkSize.
+        \param baseAllocator The allocator for allocating memory chunks.
+    */
+    MemoryPoolAllocator(size_t chunkSize = kDefaultChunkCapacity, BaseAllocator* baseAllocator = 0) : 
+        chunkHead_(0), chunk_capacity_(chunkSize), userBuffer_(0), baseAllocator_(baseAllocator), ownBaseAllocator_(0)
+    {
+    }
+
+    //! Constructor with user-supplied buffer.
+    /*! The user buffer will be used firstly. When it is full, memory pool allocates new chunk with chunk size.
+
+        The user buffer will not be deallocated when this allocator is destructed.
+
+        \param buffer User supplied buffer.
+        \param size Size of the buffer in bytes. It must at least larger than sizeof(ChunkHeader).
+        \param chunkSize The size of memory chunk. The default is kDefaultChunkSize.
+        \param baseAllocator The allocator for allocating memory chunks.
+    */
+    MemoryPoolAllocator(void *buffer, size_t size, size_t chunkSize = kDefaultChunkCapacity, BaseAllocator* baseAllocator = 0) :
+        chunkHead_(0), chunk_capacity_(chunkSize), userBuffer_(buffer), baseAllocator_(baseAllocator), ownBaseAllocator_(0)
+    {
+        RAPIDJSON_ASSERT(buffer != 0);
+        RAPIDJSON_ASSERT(size > sizeof(ChunkHeader));
+        chunkHead_ = reinterpret_cast<ChunkHeader*>(buffer);
+        chunkHead_->capacity = size - sizeof(ChunkHeader);
+        chunkHead_->size = 0;
+        chunkHead_->next = 0;
+    }
+
+    //! Destructor.
+    /*! This deallocates all memory chunks, excluding the user-supplied buffer.
+    */
+    ~MemoryPoolAllocator() {
+        Clear();
+        RAPIDJSON_DELETE(ownBaseAllocator_);
+    }
+
+    //! Deallocates all memory chunks, excluding the user-supplied buffer.
+    void Clear() {
+        while (chunkHead_ && chunkHead_ != userBuffer_) {
+            ChunkHeader* next = chunkHead_->next;
+            baseAllocator_->Free(chunkHead_);
+            chunkHead_ = next;
+        }
+        if (chunkHead_ && chunkHead_ == userBuffer_)
+            chunkHead_->size = 0; // Clear user buffer
+    }
+
+    //! Computes the total capacity of allocated memory chunks.
+    /*! \return total capacity in bytes.
+    */
+    size_t Capacity() const {
+        size_t capacity = 0;
+        for (ChunkHeader* c = chunkHead_; c != 0; c = c->next)
+            capacity += c->capacity;
+        return capacity;
+    }
+
+    //! Computes the memory blocks allocated.
+    /*! \return total used bytes.
+    */
+    size_t Size() const {
+        size_t size = 0;
+        for (ChunkHeader* c = chunkHead_; c != 0; c = c->next)
+            size += c->size;
+        return size;
+    }
+
+    //! Allocates a memory block. (concept Allocator)
+    void* Malloc(size_t size) {
+        if (!size)
+            return NULL;
+
+        size = RAPIDJSON_ALIGN(size);
+        if (chunkHead_ == 0 || chunkHead_->size + size > chunkHead_->capacity)
+            if (!AddChunk(chunk_capacity_ > size ? chunk_capacity_ : size))
+                return NULL;
+
+        void *buffer = reinterpret_cast<char *>(chunkHead_) + RAPIDJSON_ALIGN(sizeof(ChunkHeader)) + chunkHead_->size;
+        chunkHead_->size += size;
+        return buffer;
+    }
+
+    //! Resizes a memory block (concept Allocator)
+    void* Realloc(void* originalPtr, size_t originalSize, size_t newSize) {
+        if (originalPtr == 0)
+            return Malloc(newSize);
+
+        if (newSize == 0)
+            return NULL;
+
+        originalSize = RAPIDJSON_ALIGN(originalSize);
+        newSize = RAPIDJSON_ALIGN(newSize);
+
+        // Do not shrink if new size is smaller than original
+        if (originalSize >= newSize)
+            return originalPtr;
+
+        // Simply expand it if it is the last allocation and there is sufficient space
+        if (originalPtr == reinterpret_cast<char *>(chunkHead_) + RAPIDJSON_ALIGN(sizeof(ChunkHeader)) + chunkHead_->size - originalSize) {
+            size_t increment = static_cast<size_t>(newSize - originalSize);
+            if (chunkHead_->size + increment <= chunkHead_->capacity) {
+                chunkHead_->size += increment;
+                return originalPtr;
+            }
+        }
+
+        // Realloc process: allocate and copy memory, do not free original buffer.
+        if (void* newBuffer = Malloc(newSize)) {
+            if (originalSize)
+                std::memcpy(newBuffer, originalPtr, originalSize);
+            return newBuffer;
+        }
+        else
+            return NULL;
+    }
+
+    //! Frees a memory block (concept Allocator)
+    static void Free(void *ptr) { (void)ptr; } // Do nothing
+
+private:
+    //! Copy constructor is not permitted.
+    MemoryPoolAllocator(const MemoryPoolAllocator& rhs) /* = delete */;
+    //! Copy assignment operator is not permitted.
+    MemoryPoolAllocator& operator=(const MemoryPoolAllocator& rhs) /* = delete */;
+
+    //! Creates a new chunk.
+    /*! \param capacity Capacity of the chunk in bytes.
+        \return true if success.
+    */
+    bool AddChunk(size_t capacity) {
+        if (!baseAllocator_)
+            ownBaseAllocator_ = baseAllocator_ = RAPIDJSON_NEW(BaseAllocator)();
+        if (ChunkHeader* chunk = reinterpret_cast<ChunkHeader*>(baseAllocator_->Malloc(RAPIDJSON_ALIGN(sizeof(ChunkHeader)) + capacity))) {
+            chunk->capacity = capacity;
+            chunk->size = 0;
+            chunk->next = chunkHead_;
+            chunkHead_ =  chunk;
+            return true;
+        }
+        else
+            return false;
+    }
+
+    static const int kDefaultChunkCapacity = RAPIDJSON_ALLOCATOR_DEFAULT_CHUNK_CAPACITY; //!< Default chunk capacity.
+
+    //! Chunk header for perpending to each chunk.
+    /*! Chunks are stored as a singly linked list.
+    */
+    struct ChunkHeader {
+        size_t capacity;    //!< Capacity of the chunk in bytes (excluding the header itself).
+        size_t size;        //!< Current size of allocated memory in bytes.
+        ChunkHeader *next;  //!< Next chunk in the linked list.
+    };
+
+    ChunkHeader *chunkHead_;    //!< Head of the chunk linked-list. Only the head chunk serves allocation.
+    size_t chunk_capacity_;     //!< The minimum capacity of chunk when they are allocated.
+    void *userBuffer_;          //!< User supplied buffer.
+    BaseAllocator* baseAllocator_;  //!< base allocator for allocating memory chunks.
+    BaseAllocator* ownBaseAllocator_;   //!< base allocator created by this object.
+};
+
+RAPIDJSON_NAMESPACE_END
+
+#endif // RAPIDJSON_ENCODINGS_H_

vendor/github.com/Benau/go_rlottie/lottie_rapidjson_cursorstreamwrapper.h

@@ -0,0 +1,78 @@
+// Tencent is pleased to support the open source community by making RapidJSON available.
+//
+// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved.
+//
+// Licensed under the MIT License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// http://opensource.org/licenses/MIT
+//
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+#ifndef RAPIDJSON_CURSORSTREAMWRAPPER_H_
+#define RAPIDJSON_CURSORSTREAMWRAPPER_H_
+
+#include "lottie_rapidjson_stream.h"
+
+#if defined(__GNUC__)
+RAPIDJSON_DIAG_PUSH
+RAPIDJSON_DIAG_OFF(effc++)
+#endif
+
+#if defined(_MSC_VER) && _MSC_VER <= 1800
+RAPIDJSON_DIAG_PUSH
+RAPIDJSON_DIAG_OFF(4702)  // unreachable code
+RAPIDJSON_DIAG_OFF(4512)  // assignment operator could not be generated
+#endif
+
+RAPIDJSON_NAMESPACE_BEGIN
+
+
+//! Cursor stream wrapper for counting line and column number if error exists.
+/*!
+    \tparam InputStream     Any stream that implements Stream Concept
+*/
+template <typename InputStream, typename Encoding = UTF8<> >
+class CursorStreamWrapper : public GenericStreamWrapper<InputStream, Encoding> {
+public:
+    typedef typename Encoding::Ch Ch;
+
+    CursorStreamWrapper(InputStream& is):
+        GenericStreamWrapper<InputStream, Encoding>(is), line_(1), col_(0) {}
+
+    // counting line and column number
+    Ch Take() {
+        Ch ch = this->is_.Take();
+        if(ch == '\n') {
+            line_ ++;
+            col_ = 0;
+        } else {
+            col_ ++;
+        }
+        return ch;
+    }
+
+    //! Get the error line number, if error exists.
+    size_t GetLine() const { return line_; }
+    //! Get the error column number, if error exists.
+    size_t GetColumn() const { return col_; }
+
+private:
+    size_t line_;   //!< Current Line
+    size_t col_;    //!< Current Column
+};
+
+#if defined(_MSC_VER) && _MSC_VER <= 1800
+RAPIDJSON_DIAG_POP
+#endif
+
+#if defined(__GNUC__)
+RAPIDJSON_DIAG_POP
+#endif
+
+RAPIDJSON_NAMESPACE_END
+
+#endif // RAPIDJSON_CURSORSTREAMWRAPPER_H_

vendor/github.com/Benau/go_rlottie/lottie_rapidjson_encodedstream.h

@@ -0,0 +1,299 @@
+// Tencent is pleased to support the open source community by making RapidJSON available.
+// 
+// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved.
+//
+// Licensed under the MIT License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// http://opensource.org/licenses/MIT
+//
+// Unless required by applicable law or agreed to in writing, software distributed 
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the 
+// specific language governing permissions and limitations under the License.
+
+#ifndef RAPIDJSON_ENCODEDSTREAM_H_
+#define RAPIDJSON_ENCODEDSTREAM_H_
+
+#include "lottie_rapidjson_stream.h"
+#include "lottie_rapidjson_memorystream.h"
+
+#ifdef __GNUC__
+RAPIDJSON_DIAG_PUSH
+RAPIDJSON_DIAG_OFF(effc++)
+#endif
+
+#ifdef __clang__
+RAPIDJSON_DIAG_PUSH
+RAPIDJSON_DIAG_OFF(padded)
+#endif
+
+RAPIDJSON_NAMESPACE_BEGIN
+
+//! Input byte stream wrapper with a statically bound encoding.
+/*!
+    \tparam Encoding The interpretation of encoding of the stream. Either UTF8, UTF16LE, UTF16BE, UTF32LE, UTF32BE.
+    \tparam InputByteStream Type of input byte stream. For example, FileReadStream.
+*/
+template <typename Encoding, typename InputByteStream>
+class EncodedInputStream {
+    RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1);
+public:
+    typedef typename Encoding::Ch Ch;
+
+    EncodedInputStream(InputByteStream& is) : is_(is) { 
+        current_ = Encoding::TakeBOM(is_);
+    }
+
+    Ch Peek() const { return current_; }
+    Ch Take() { Ch c = current_; current_ = Encoding::Take(is_); return c; }
+    size_t Tell() const { return is_.Tell(); }
+
+    // Not implemented
+    void Put(Ch) { RAPIDJSON_ASSERT(false); }
+    void Flush() { RAPIDJSON_ASSERT(false); } 
+    Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; }
+    size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; }
+
+private:
+    EncodedInputStream(const EncodedInputStream&);
+    EncodedInputStream& operator=(const EncodedInputStream&);
+
+    InputByteStream& is_;
+    Ch current_;
+};
+
+//! Specialized for UTF8 MemoryStream.
+template <>
+class EncodedInputStream<UTF8<>, MemoryStream> {
+public:
+    typedef UTF8<>::Ch Ch;
+
+    EncodedInputStream(MemoryStream& is) : is_(is) {
+        if (static_cast<unsigned char>(is_.Peek()) == 0xEFu) is_.Take();
+        if (static_cast<unsigned char>(is_.Peek()) == 0xBBu) is_.Take();
+        if (static_cast<unsigned char>(is_.Peek()) == 0xBFu) is_.Take();
+    }
+    Ch Peek() const { return is_.Peek(); }
+    Ch Take() { return is_.Take(); }
+    size_t Tell() const { return is_.Tell(); }
+
+    // Not implemented
+    void Put(Ch) {}
+    void Flush() {} 
+    Ch* PutBegin() { return 0; }
+    size_t PutEnd(Ch*) { return 0; }
+
+    MemoryStream& is_;
+
+private:
+    EncodedInputStream(const EncodedInputStream&);
+    EncodedInputStream& operator=(const EncodedInputStream&);
+};
+
+//! Output byte stream wrapper with statically bound encoding.
+/*!
+    \tparam Encoding The interpretation of encoding of the stream. Either UTF8, UTF16LE, UTF16BE, UTF32LE, UTF32BE.
+    \tparam OutputByteStream Type of input byte stream. For example, FileWriteStream.
+*/
+template <typename Encoding, typename OutputByteStream>
+class EncodedOutputStream {
+    RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1);
+public:
+    typedef typename Encoding::Ch Ch;
+
+    EncodedOutputStream(OutputByteStream& os, bool putBOM = true) : os_(os) { 
+        if (putBOM)
+            Encoding::PutBOM(os_);
+    }
+
+    void Put(Ch c) { Encoding::Put(os_, c);  }
+    void Flush() { os_.Flush(); }
+
+    // Not implemented
+    Ch Peek() const { RAPIDJSON_ASSERT(false); return 0;}
+    Ch Take() { RAPIDJSON_ASSERT(false); return 0;}
+    size_t Tell() const { RAPIDJSON_ASSERT(false);  return 0; }
+    Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; }
+    size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; }
+
+private:
+    EncodedOutputStream(const EncodedOutputStream&);
+    EncodedOutputStream& operator=(const EncodedOutputStream&);
+
+    OutputByteStream& os_;
+};
+
+#define RAPIDJSON_ENCODINGS_FUNC(x) UTF8<Ch>::x, UTF16LE<Ch>::x, UTF16BE<Ch>::x, UTF32LE<Ch>::x, UTF32BE<Ch>::x
+
+//! Input stream wrapper with dynamically bound encoding and automatic encoding detection.
+/*!
+    \tparam CharType Type of character for reading.
+    \tparam InputByteStream type of input byte stream to be wrapped.
+*/
+template <typename CharType, typename InputByteStream>
+class AutoUTFInputStream {
+    RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1);
+public:
+    typedef CharType Ch;
+
+    //! Constructor.
+    /*!
+        \param is input stream to be wrapped.
+        \param type UTF encoding type if it is not detected from the stream.
+    */
+    AutoUTFInputStream(InputByteStream& is, UTFType type = kUTF8) : is_(&is), type_(type), hasBOM_(false) {
+        RAPIDJSON_ASSERT(type >= kUTF8 && type <= kUTF32BE);        
+        DetectType();
+        static const TakeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Take) };
+        takeFunc_ = f[type_];
+        current_ = takeFunc_(*is_);
+    }
+
+    UTFType GetType() const { return type_; }
+    bool HasBOM() const { return hasBOM_; }
+
+    Ch Peek() const { return current_; }
+    Ch Take() { Ch c = current_; current_ = takeFunc_(*is_); return c; }
+    size_t Tell() const { return is_->Tell(); }
+
+    // Not implemented
+    void Put(Ch) { RAPIDJSON_ASSERT(false); }
+    void Flush() { RAPIDJSON_ASSERT(false); } 
+    Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; }
+    size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; }
+
+private:
+    AutoUTFInputStream(const AutoUTFInputStream&);
+    AutoUTFInputStream& operator=(const AutoUTFInputStream&);
+
+    // Detect encoding type with BOM or RFC 4627
+    void DetectType() {
+        // BOM (Byte Order Mark):
+        // 00 00 FE FF  UTF-32BE
+        // FF FE 00 00  UTF-32LE
+        // FE FF        UTF-16BE
+        // FF FE        UTF-16LE
+        // EF BB BF     UTF-8
+
+        const unsigned char* c = reinterpret_cast<const unsigned char *>(is_->Peek4());
+        if (!c)
+            return;
+
+        unsigned bom = static_cast<unsigned>(c[0] | (c[1] << 8) | (c[2] << 16) | (c[3] << 24));
+        hasBOM_ = false;
+        if (bom == 0xFFFE0000)                  { type_ = kUTF32BE; hasBOM_ = true; is_->Take(); is_->Take(); is_->Take(); is_->Take(); }
+        else if (bom == 0x0000FEFF)             { type_ = kUTF32LE; hasBOM_ = true; is_->Take(); is_->Take(); is_->Take(); is_->Take(); }
+        else if ((bom & 0xFFFF) == 0xFFFE)      { type_ = kUTF16BE; hasBOM_ = true; is_->Take(); is_->Take();                           }
+        else if ((bom & 0xFFFF) == 0xFEFF)      { type_ = kUTF16LE; hasBOM_ = true; is_->Take(); is_->Take();                           }
+        else if ((bom & 0xFFFFFF) == 0xBFBBEF)  { type_ = kUTF8;    hasBOM_ = true; is_->Take(); is_->Take(); is_->Take();              }
+
+        // RFC 4627: Section 3
+        // "Since the first two characters of a JSON text will always be ASCII
+        // characters [RFC0020], it is possible to determine whether an octet
+        // stream is UTF-8, UTF-16 (BE or LE), or UTF-32 (BE or LE) by looking
+        // at the pattern of nulls in the first four octets."
+        // 00 00 00 xx  UTF-32BE
+        // 00 xx 00 xx  UTF-16BE
+        // xx 00 00 00  UTF-32LE
+        // xx 00 xx 00  UTF-16LE
+        // xx xx xx xx  UTF-8
+
+        if (!hasBOM_) {
+            int pattern = (c[0] ? 1 : 0) | (c[1] ? 2 : 0) | (c[2] ? 4 : 0) | (c[3] ? 8 : 0);
+            switch (pattern) {
+            case 0x08: type_ = kUTF32BE; break;
+            case 0x0A: type_ = kUTF16BE; break;
+            case 0x01: type_ = kUTF32LE; break;
+            case 0x05: type_ = kUTF16LE; break;
+            case 0x0F: type_ = kUTF8;    break;
+            default: break; // Use type defined by user.
+            }
+        }
+
+        // Runtime check whether the size of character type is sufficient. It only perform checks with assertion.
+        if (type_ == kUTF16LE || type_ == kUTF16BE) RAPIDJSON_ASSERT(sizeof(Ch) >= 2);
+        if (type_ == kUTF32LE || type_ == kUTF32BE) RAPIDJSON_ASSERT(sizeof(Ch) >= 4);
+    }
+
+    typedef Ch (*TakeFunc)(InputByteStream& is);
+    InputByteStream* is_;
+    UTFType type_;
+    Ch current_;
+    TakeFunc takeFunc_;
+    bool hasBOM_;
+};
+
+//! Output stream wrapper with dynamically bound encoding and automatic encoding detection.
+/*!
+    \tparam CharType Type of character for writing.
+    \tparam OutputByteStream type of output byte stream to be wrapped.
+*/
+template <typename CharType, typename OutputByteStream>
+class AutoUTFOutputStream {
+    RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1);
+public:
+    typedef CharType Ch;
+
+    //! Constructor.
+    /*!
+        \param os output stream to be wrapped.
+        \param type UTF encoding type.
+        \param putBOM Whether to write BOM at the beginning of the stream.
+    */
+    AutoUTFOutputStream(OutputByteStream& os, UTFType type, bool putBOM) : os_(&os), type_(type) {
+        RAPIDJSON_ASSERT(type >= kUTF8 && type <= kUTF32BE);
+
+        // Runtime check whether the size of character type is sufficient. It only perform checks with assertion.
+        if (type_ == kUTF16LE || type_ == kUTF16BE) RAPIDJSON_ASSERT(sizeof(Ch) >= 2);
+        if (type_ == kUTF32LE || type_ == kUTF32BE) RAPIDJSON_ASSERT(sizeof(Ch) >= 4);
+
+        static const PutFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Put) };
+        putFunc_ = f[type_];
+
+        if (putBOM)
+            PutBOM();
+    }
+
+    UTFType GetType() const { return type_; }
+
+    void Put(Ch c) { putFunc_(*os_, c); }
+    void Flush() { os_->Flush(); } 
+
+    // Not implemented
+    Ch Peek() const { RAPIDJSON_ASSERT(false); return 0;}
+    Ch Take() { RAPIDJSON_ASSERT(false); return 0;}
+    size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; }
+    Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; }
+    size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; }
+
+private:
+    AutoUTFOutputStream(const AutoUTFOutputStream&);
+    AutoUTFOutputStream& operator=(const AutoUTFOutputStream&);
+
+    void PutBOM() { 
+        typedef void (*PutBOMFunc)(OutputByteStream&);
+        static const PutBOMFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(PutBOM) };
+        f[type_](*os_);
+    }
+
+    typedef void (*PutFunc)(OutputByteStream&, Ch);
+
+    OutputByteStream* os_;
+    UTFType type_;
+    PutFunc putFunc_;
+};
+
+#undef RAPIDJSON_ENCODINGS_FUNC
+
+RAPIDJSON_NAMESPACE_END
+
+#ifdef __clang__
+RAPIDJSON_DIAG_POP
+#endif
+
+#ifdef __GNUC__
+RAPIDJSON_DIAG_POP
+#endif
+
+#endif // RAPIDJSON_FILESTREAM_H_

vendor/github.com/Benau/go_rlottie/lottie_rapidjson_encodings.h

@@ -0,0 +1,716 @@
+// Tencent is pleased to support the open source community by making RapidJSON available.
+// 
+// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved.
+//
+// Licensed under the MIT License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// http://opensource.org/licenses/MIT
+//
+// Unless required by applicable law or agreed to in writing, software distributed 
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the 
+// specific language governing permissions and limitations under the License.
+
+#ifndef RAPIDJSON_ENCODINGS_H_
+#define RAPIDJSON_ENCODINGS_H_
+
+#include "lottie_rapidjson_rapidjson.h"
+
+#if defined(_MSC_VER) && !defined(__clang__)
+RAPIDJSON_DIAG_PUSH
+RAPIDJSON_DIAG_OFF(4244) // conversion from 'type1' to 'type2', possible loss of data
+RAPIDJSON_DIAG_OFF(4702)  // unreachable code
+#elif defined(__GNUC__)
+RAPIDJSON_DIAG_PUSH
+RAPIDJSON_DIAG_OFF(effc++)
+RAPIDJSON_DIAG_OFF(overflow)
+#endif
+
+RAPIDJSON_NAMESPACE_BEGIN
+
+///////////////////////////////////////////////////////////////////////////////
+// Encoding
+
+/*! \class rapidjson::Encoding
+    \brief Concept for encoding of Unicode characters.
+
+\code
+concept Encoding {
+    typename Ch;    //! Type of character. A "character" is actually a code unit in unicode's definition.
+
+    enum { supportUnicode = 1 }; // or 0 if not supporting unicode
+
+    //! \brief Encode a Unicode codepoint to an output stream.
+    //! \param os Output stream.
+    //! \param codepoint An unicode codepoint, ranging from 0x0 to 0x10FFFF inclusively.
+    template<typename OutputStream>
+    static void Encode(OutputStream& os, unsigned codepoint);
+
+    //! \brief Decode a Unicode codepoint from an input stream.
+    //! \param is Input stream.
+    //! \param codepoint Output of the unicode codepoint.
+    //! \return true if a valid codepoint can be decoded from the stream.
+    template <typename InputStream>
+    static bool Decode(InputStream& is, unsigned* codepoint);
+
+    //! \brief Validate one Unicode codepoint from an encoded stream.
+    //! \param is Input stream to obtain codepoint.
+    //! \param os Output for copying one codepoint.
+    //! \return true if it is valid.
+    //! \note This function just validating and copying the codepoint without actually decode it.
+    template <typename InputStream, typename OutputStream>
+    static bool Validate(InputStream& is, OutputStream& os);
+
+    // The following functions are deal with byte streams.
+
+    //! Take a character from input byte stream, skip BOM if exist.
+    template <typename InputByteStream>
+    static CharType TakeBOM(InputByteStream& is);
+
+    //! Take a character from input byte stream.
+    template <typename InputByteStream>
+    static Ch Take(InputByteStream& is);
+
+    //! Put BOM to output byte stream.
+    template <typename OutputByteStream>
+    static void PutBOM(OutputByteStream& os);
+
+    //! Put a character to output byte stream.
+    template <typename OutputByteStream>
+    static void Put(OutputByteStream& os, Ch c);
+};
+\endcode
+*/
+
+///////////////////////////////////////////////////////////////////////////////
+// UTF8
+
+//! UTF-8 encoding.
+/*! http://en.wikipedia.org/wiki/UTF-8
+    http://tools.ietf.org/html/rfc3629
+    \tparam CharType Code unit for storing 8-bit UTF-8 data. Default is char.
+    \note implements Encoding concept
+*/
+template<typename CharType = char>
+struct UTF8 {
+    typedef CharType Ch;
+
+    enum { supportUnicode = 1 };
+
+    template<typename OutputStream>
+    static void Encode(OutputStream& os, unsigned codepoint) {
+        if (codepoint <= 0x7F) 
+            os.Put(static_cast<Ch>(codepoint & 0xFF));
+        else if (codepoint <= 0x7FF) {
+            os.Put(static_cast<Ch>(0xC0 | ((codepoint >> 6) & 0xFF)));
+            os.Put(static_cast<Ch>(0x80 | ((codepoint & 0x3F))));
+        }
+        else if (codepoint <= 0xFFFF) {
+            os.Put(static_cast<Ch>(0xE0 | ((codepoint >> 12) & 0xFF)));
+            os.Put(static_cast<Ch>(0x80 | ((codepoint >> 6) & 0x3F)));
+            os.Put(static_cast<Ch>(0x80 | (codepoint & 0x3F)));
+        }
+        else {
+            RAPIDJSON_ASSERT(codepoint <= 0x10FFFF);
+            os.Put(static_cast<Ch>(0xF0 | ((codepoint >> 18) & 0xFF)));
+            os.Put(static_cast<Ch>(0x80 | ((codepoint >> 12) & 0x3F)));
+            os.Put(static_cast<Ch>(0x80 | ((codepoint >> 6) & 0x3F)));
+            os.Put(static_cast<Ch>(0x80 | (codepoint & 0x3F)));
+        }
+    }
+
+    template<typename OutputStream>
+    static void EncodeUnsafe(OutputStream& os, unsigned codepoint) {
+        if (codepoint <= 0x7F) 
+            PutUnsafe(os, static_cast<Ch>(codepoint & 0xFF));
+        else if (codepoint <= 0x7FF) {
+            PutUnsafe(os, static_cast<Ch>(0xC0 | ((codepoint >> 6) & 0xFF)));
+            PutUnsafe(os, static_cast<Ch>(0x80 | ((codepoint & 0x3F))));
+        }
+        else if (codepoint <= 0xFFFF) {
+            PutUnsafe(os, static_cast<Ch>(0xE0 | ((codepoint >> 12) & 0xFF)));
+            PutUnsafe(os, static_cast<Ch>(0x80 | ((codepoint >> 6) & 0x3F)));
+            PutUnsafe(os, static_cast<Ch>(0x80 | (codepoint & 0x3F)));
+        }
+        else {
+            RAPIDJSON_ASSERT(codepoint <= 0x10FFFF);
+            PutUnsafe(os, static_cast<Ch>(0xF0 | ((codepoint >> 18) & 0xFF)));
+            PutUnsafe(os, static_cast<Ch>(0x80 | ((codepoint >> 12) & 0x3F)));
+            PutUnsafe(os, static_cast<Ch>(0x80 | ((codepoint >> 6) & 0x3F)));
+            PutUnsafe(os, static_cast<Ch>(0x80 | (codepoint & 0x3F)));
+        }
+    }
+
+    template <typename InputStream>
+    static bool Decode(InputStream& is, unsigned* codepoint) {
+#define RAPIDJSON_COPY() c = is.Take(); *codepoint = (*codepoint << 6) | (static_cast<unsigned char>(c) & 0x3Fu)
+#define RAPIDJSON_TRANS(mask) result &= ((GetRange(static_cast<unsigned char>(c)) & mask) != 0)
+#define RAPIDJSON_TAIL() RAPIDJSON_COPY(); RAPIDJSON_TRANS(0x70)
+        typename InputStream::Ch c = is.Take();
+        if (!(c & 0x80)) {
+            *codepoint = static_cast<unsigned char>(c);
+            return true;
+        }
+
+        unsigned char type = GetRange(static_cast<unsigned char>(c));
+        if (type >= 32) {
+            *codepoint = 0;
+        } else {
+            *codepoint = (0xFFu >> type) & static_cast<unsigned char>(c);
+        }
+        bool result = true;
+        switch (type) {
+        case 2: RAPIDJSON_TAIL(); return result;
+        case 3: RAPIDJSON_TAIL(); RAPIDJSON_TAIL(); return result;
+        case 4: RAPIDJSON_COPY(); RAPIDJSON_TRANS(0x50); RAPIDJSON_TAIL(); return result;
+        case 5: RAPIDJSON_COPY(); RAPIDJSON_TRANS(0x10); RAPIDJSON_TAIL(); RAPIDJSON_TAIL(); return result;
+        case 6: RAPIDJSON_TAIL(); RAPIDJSON_TAIL(); RAPIDJSON_TAIL(); return result;
+        case 10: RAPIDJSON_COPY(); RAPIDJSON_TRANS(0x20); RAPIDJSON_TAIL(); return result;
+        case 11: RAPIDJSON_COPY(); RAPIDJSON_TRANS(0x60); RAPIDJSON_TAIL(); RAPIDJSON_TAIL(); return result;
+        default: return false;
+        }
+#undef RAPIDJSON_COPY
+#undef RAPIDJSON_TRANS
+#undef RAPIDJSON_TAIL
+    }
+
+    template <typename InputStream, typename OutputStream>
+    static bool Validate(InputStream& is, OutputStream& os) {
+#define RAPIDJSON_COPY() os.Put(c = is.Take())
+#define RAPIDJSON_TRANS(mask) result &= ((GetRange(static_cast<unsigned char>(c)) & mask) != 0)
+#define RAPIDJSON_TAIL() RAPIDJSON_COPY(); RAPIDJSON_TRANS(0x70)
+        Ch c;
+        RAPIDJSON_COPY();
+        if (!(c & 0x80))
+            return true;
+
+        bool result = true;
+        switch (GetRange(static_cast<unsigned char>(c))) {
+        case 2: RAPIDJSON_TAIL(); return result;
+        case 3: RAPIDJSON_TAIL(); RAPIDJSON_TAIL(); return result;
+        case 4: RAPIDJSON_COPY(); RAPIDJSON_TRANS(0x50); RAPIDJSON_TAIL(); return result;
+        case 5: RAPIDJSON_COPY(); RAPIDJSON_TRANS(0x10); RAPIDJSON_TAIL(); RAPIDJSON_TAIL(); return result;
+        case 6: RAPIDJSON_TAIL(); RAPIDJSON_TAIL(); RAPIDJSON_TAIL(); return result;
+        case 10: RAPIDJSON_COPY(); RAPIDJSON_TRANS(0x20); RAPIDJSON_TAIL(); return result;
+        case 11: RAPIDJSON_COPY(); RAPIDJSON_TRANS(0x60); RAPIDJSON_TAIL(); RAPIDJSON_TAIL(); return result;
+        default: return false;
+        }
+#undef RAPIDJSON_COPY
+#undef RAPIDJSON_TRANS
+#undef RAPIDJSON_TAIL
+    }
+
+    static unsigned char GetRange(unsigned char c) {
+        // Referring to DFA of http://bjoern.hoehrmann.de/utf-8/decoder/dfa/
+        // With new mapping 1 -> 0x10, 7 -> 0x20, 9 -> 0x40, such that AND operation can test multiple types.
+        static const unsigned char type[] = {
+            0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+            0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+            0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+            0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+            0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,
+            0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,
+            0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,
+            0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,
+            8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2,  2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
+            10,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3, 11,6,6,6,5,8,8,8,8,8,8,8,8,8,8,8,
+        };
+        return type[c];
+    }
+
+    template <typename InputByteStream>
+    static CharType TakeBOM(InputByteStream& is) {
+        RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1);
+        typename InputByteStream::Ch c = Take(is);
+        if (static_cast<unsigned char>(c) != 0xEFu) return c;
+        c = is.Take();
+        if (static_cast<unsigned char>(c) != 0xBBu) return c;
+        c = is.Take();
+        if (static_cast<unsigned char>(c) != 0xBFu) return c;
+        c = is.Take();
+        return c;
+    }
+
+    template <typename InputByteStream>
+    static Ch Take(InputByteStream& is) {
+        RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1);
+        return static_cast<Ch>(is.Take());
+    }
+
+    template <typename OutputByteStream>
+    static void PutBOM(OutputByteStream& os) {
+        RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1);
+        os.Put(static_cast<typename OutputByteStream::Ch>(0xEFu));
+        os.Put(static_cast<typename OutputByteStream::Ch>(0xBBu));
+        os.Put(static_cast<typename OutputByteStream::Ch>(0xBFu));
+    }
+
+    template <typename OutputByteStream>
+    static void Put(OutputByteStream& os, Ch c) {
+        RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1);
+        os.Put(static_cast<typename OutputByteStream::Ch>(c));
+    }
+};
+
+///////////////////////////////////////////////////////////////////////////////
+// UTF16
+
+//! UTF-16 encoding.
+/*! http://en.wikipedia.org/wiki/UTF-16
+    http://tools.ietf.org/html/rfc2781
+    \tparam CharType Type for storing 16-bit UTF-16 data. Default is wchar_t. C++11 may use char16_t instead.
+    \note implements Encoding concept
+
+    \note For in-memory access, no need to concern endianness. The code units and code points are represented by CPU's endianness.
+    For streaming, use UTF16LE and UTF16BE, which handle endianness.
+*/
+template<typename CharType = wchar_t>
+struct UTF16 {
+    typedef CharType Ch;
+    RAPIDJSON_STATIC_ASSERT(sizeof(Ch) >= 2);
+
+    enum { supportUnicode = 1 };
+
+    template<typename OutputStream>
+    static void Encode(OutputStream& os, unsigned codepoint) {
+        RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 2);
+        if (codepoint <= 0xFFFF) {
+            RAPIDJSON_ASSERT(codepoint < 0xD800 || codepoint > 0xDFFF); // Code point itself cannot be surrogate pair 
+            os.Put(static_cast<typename OutputStream::Ch>(codepoint));
+        }
+        else {
+            RAPIDJSON_ASSERT(codepoint <= 0x10FFFF);
+            unsigned v = codepoint - 0x10000;
+            os.Put(static_cast<typename OutputStream::Ch>((v >> 10) | 0xD800));
+            os.Put(static_cast<typename OutputStream::Ch>((v & 0x3FF) | 0xDC00));
+        }
+    }
+
+
+    template<typename OutputStream>
+    static void EncodeUnsafe(OutputStream& os, unsigned codepoint) {
+        RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 2);
+        if (codepoint <= 0xFFFF) {
+            RAPIDJSON_ASSERT(codepoint < 0xD800 || codepoint > 0xDFFF); // Code point itself cannot be surrogate pair 
+            PutUnsafe(os, static_cast<typename OutputStream::Ch>(codepoint));
+        }
+        else {
+            RAPIDJSON_ASSERT(codepoint <= 0x10FFFF);
+            unsigned v = codepoint - 0x10000;
+            PutUnsafe(os, static_cast<typename OutputStream::Ch>((v >> 10) | 0xD800));
+            PutUnsafe(os, static_cast<typename OutputStream::Ch>((v & 0x3FF) | 0xDC00));
+        }
+    }
+
+    template <typename InputStream>
+    static bool Decode(InputStream& is, unsigned* codepoint) {
+        RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 2);
+        typename InputStream::Ch c = is.Take();
+        if (c < 0xD800 || c > 0xDFFF) {
+            *codepoint = static_cast<unsigned>(c);
+            return true;
+        }
+        else if (c <= 0xDBFF) {
+            *codepoint = (static_cast<unsigned>(c) & 0x3FF) << 10;
+            c = is.Take();
+            *codepoint |= (static_cast<unsigned>(c) & 0x3FF);
+            *codepoint += 0x10000;
+            return c >= 0xDC00 && c <= 0xDFFF;
+        }
+        return false;
+    }
+
+    template <typename InputStream, typename OutputStream>
+    static bool Validate(InputStream& is, OutputStream& os) {
+        RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 2);
+        RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 2);
+        typename InputStream::Ch c;
+        os.Put(static_cast<typename OutputStream::Ch>(c = is.Take()));
+        if (c < 0xD800 || c > 0xDFFF)
+            return true;
+        else if (c <= 0xDBFF) {
+            os.Put(c = is.Take());
+            return c >= 0xDC00 && c <= 0xDFFF;
+        }
+        return false;
+    }
+};
+
+//! UTF-16 little endian encoding.
+template<typename CharType = wchar_t>
+struct UTF16LE : UTF16<CharType> {
+    template <typename InputByteStream>
+    static CharType TakeBOM(InputByteStream& is) {
+        RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1);
+        CharType c = Take(is);
+        return static_cast<uint16_t>(c) == 0xFEFFu ? Take(is) : c;
+    }
+
+    template <typename InputByteStream>
+    static CharType Take(InputByteStream& is) {
+        RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1);
+        unsigned c = static_cast<uint8_t>(is.Take());
+        c |= static_cast<unsigned>(static_cast<uint8_t>(is.Take())) << 8;
+        return static_cast<CharType>(c);
+    }
+
+    template <typename OutputByteStream>
+    static void PutBOM(OutputByteStream& os) {
+        RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1);
+        os.Put(static_cast<typename OutputByteStream::Ch>(0xFFu));
+        os.Put(static_cast<typename OutputByteStream::Ch>(0xFEu));
+    }
+
+    template <typename OutputByteStream>
+    static void Put(OutputByteStream& os, CharType c) {
+        RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1);
+        os.Put(static_cast<typename OutputByteStream::Ch>(static_cast<unsigned>(c) & 0xFFu));
+        os.Put(static_cast<typename OutputByteStream::Ch>((static_cast<unsigned>(c) >> 8) & 0xFFu));
+    }
+};
+
+//! UTF-16 big endian encoding.
+template<typename CharType = wchar_t>
+struct UTF16BE : UTF16<CharType> {
+    template <typename InputByteStream>
+    static CharType TakeBOM(InputByteStream& is) {
+        RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1);
+        CharType c = Take(is);
+        return static_cast<uint16_t>(c) == 0xFEFFu ? Take(is) : c;
+    }
+
+    template <typename InputByteStream>
+    static CharType Take(InputByteStream& is) {
+        RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1);
+        unsigned c = static_cast<unsigned>(static_cast<uint8_t>(is.Take())) << 8;
+        c |= static_cast<unsigned>(static_cast<uint8_t>(is.Take()));
+        return static_cast<CharType>(c);
+    }
+
+    template <typename OutputByteStream>
+    static void PutBOM(OutputByteStream& os) {
+        RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1);
+        os.Put(static_cast<typename OutputByteStream::Ch>(0xFEu));
+        os.Put(static_cast<typename OutputByteStream::Ch>(0xFFu));
+    }
+
+    template <typename OutputByteStream>
+    static void Put(OutputByteStream& os, CharType c) {
+        RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1);
+        os.Put(static_cast<typename OutputByteStream::Ch>((static_cast<unsigned>(c) >> 8) & 0xFFu));
+        os.Put(static_cast<typename OutputByteStream::Ch>(static_cast<unsigned>(c) & 0xFFu));
+    }
+};
+
+///////////////////////////////////////////////////////////////////////////////
+// UTF32
+
+//! UTF-32 encoding. 
+/*! http://en.wikipedia.org/wiki/UTF-32
+    \tparam CharType Type for storing 32-bit UTF-32 data. Default is unsigned. C++11 may use char32_t instead.
+    \note implements Encoding concept
+
+    \note For in-memory access, no need to concern endianness. The code units and code points are represented by CPU's endianness.
+    For streaming, use UTF32LE and UTF32BE, which handle endianness.
+*/
+template<typename CharType = unsigned>
+struct UTF32 {
+    typedef CharType Ch;
+    RAPIDJSON_STATIC_ASSERT(sizeof(Ch) >= 4);
+
+    enum { supportUnicode = 1 };
+
+    template<typename OutputStream>
+    static void Encode(OutputStream& os, unsigned codepoint) {
+        RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 4);
+        RAPIDJSON_ASSERT(codepoint <= 0x10FFFF);
+        os.Put(codepoint);
+    }
+
+    template<typename OutputStream>
+    static void EncodeUnsafe(OutputStream& os, unsigned codepoint) {
+        RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 4);
+        RAPIDJSON_ASSERT(codepoint <= 0x10FFFF);
+        PutUnsafe(os, codepoint);
+    }
+
+    template <typename InputStream>
+    static bool Decode(InputStream& is, unsigned* codepoint) {
+        RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 4);
+        Ch c = is.Take();
+        *codepoint = c;
+        return c <= 0x10FFFF;
+    }
+
+    template <typename InputStream, typename OutputStream>
+    static bool Validate(InputStream& is, OutputStream& os) {
+        RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 4);
+        Ch c;
+        os.Put(c = is.Take());
+        return c <= 0x10FFFF;
+    }
+};
+
+//! UTF-32 little endian enocoding.
+template<typename CharType = unsigned>
+struct UTF32LE : UTF32<CharType> {
+    template <typename InputByteStream>
+    static CharType TakeBOM(InputByteStream& is) {
+        RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1);
+        CharType c = Take(is);
+        return static_cast<uint32_t>(c) == 0x0000FEFFu ? Take(is) : c;
+    }
+
+    template <typename InputByteStream>
+    static CharType Take(InputByteStream& is) {
+        RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1);
+        unsigned c = static_cast<uint8_t>(is.Take());
+        c |= static_cast<unsigned>(static_cast<uint8_t>(is.Take())) << 8;
+        c |= static_cast<unsigned>(static_cast<uint8_t>(is.Take())) << 16;
+        c |= static_cast<unsigned>(static_cast<uint8_t>(is.Take())) << 24;
+        return static_cast<CharType>(c);
+    }
+
+    template <typename OutputByteStream>
+    static void PutBOM(OutputByteStream& os) {
+        RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1);
+        os.Put(static_cast<typename OutputByteStream::Ch>(0xFFu));
+        os.Put(static_cast<typename OutputByteStream::Ch>(0xFEu));
+        os.Put(static_cast<typename OutputByteStream::Ch>(0x00u));
+        os.Put(static_cast<typename OutputByteStream::Ch>(0x00u));
+    }
+
+    template <typename OutputByteStream>
+    static void Put(OutputByteStream& os, CharType c) {
+        RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1);
+        os.Put(static_cast<typename OutputByteStream::Ch>(c & 0xFFu));
+        os.Put(static_cast<typename OutputByteStream::Ch>((c >> 8) & 0xFFu));
+        os.Put(static_cast<typename OutputByteStream::Ch>((c >> 16) & 0xFFu));
+        os.Put(static_cast<typename OutputByteStream::Ch>((c >> 24) & 0xFFu));
+    }
+};
+
+//! UTF-32 big endian encoding.
+template<typename CharType = unsigned>
+struct UTF32BE : UTF32<CharType> {
+    template <typename InputByteStream>
+    static CharType TakeBOM(InputByteStream& is) {
+        RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1);
+        CharType c = Take(is);
+        return static_cast<uint32_t>(c) == 0x0000FEFFu ? Take(is) : c; 
+    }
+
+    template <typename InputByteStream>
+    static CharType Take(InputByteStream& is) {
+        RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1);
+        unsigned c = static_cast<unsigned>(static_cast<uint8_t>(is.Take())) << 24;
+        c |= static_cast<unsigned>(static_cast<uint8_t>(is.Take())) << 16;
+        c |= static_cast<unsigned>(static_cast<uint8_t>(is.Take())) << 8;
+        c |= static_cast<unsigned>(static_cast<uint8_t>(is.Take()));
+        return static_cast<CharType>(c);
+    }
+
+    template <typename OutputByteStream>
+    static void PutBOM(OutputByteStream& os) {
+        RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1);
+        os.Put(static_cast<typename OutputByteStream::Ch>(0x00u));
+        os.Put(static_cast<typename OutputByteStream::Ch>(0x00u));
+        os.Put(static_cast<typename OutputByteStream::Ch>(0xFEu));
+        os.Put(static_cast<typename OutputByteStream::Ch>(0xFFu));
+    }
+
+    template <typename OutputByteStream>
+    static void Put(OutputByteStream& os, CharType c) {
+        RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1);
+        os.Put(static_cast<typename OutputByteStream::Ch>((c >> 24) & 0xFFu));
+        os.Put(static_cast<typename OutputByteStream::Ch>((c >> 16) & 0xFFu));
+        os.Put(static_cast<typename OutputByteStream::Ch>((c >> 8) & 0xFFu));
+        os.Put(static_cast<typename OutputByteStream::Ch>(c & 0xFFu));
+    }
+};
+
+///////////////////////////////////////////////////////////////////////////////
+// ASCII
+
+//! ASCII encoding.
+/*! http://en.wikipedia.org/wiki/ASCII
+    \tparam CharType Code unit for storing 7-bit ASCII data. Default is char.
+    \note implements Encoding concept
+*/
+template<typename CharType = char>
+struct ASCII {
+    typedef CharType Ch;
+
+    enum { supportUnicode = 0 };
+
+    template<typename OutputStream>
+    static void Encode(OutputStream& os, unsigned codepoint) {
+        RAPIDJSON_ASSERT(codepoint <= 0x7F);
+        os.Put(static_cast<Ch>(codepoint & 0xFF));
+    }
+
+    template<typename OutputStream>
+    static void EncodeUnsafe(OutputStream& os, unsigned codepoint) {
+        RAPIDJSON_ASSERT(codepoint <= 0x7F);
+        PutUnsafe(os, static_cast<Ch>(codepoint & 0xFF));
+    }
+
+    template <typename InputStream>
+    static bool Decode(InputStream& is, unsigned* codepoint) {
+        uint8_t c = static_cast<uint8_t>(is.Take());
+        *codepoint = c;
+        return c <= 0X7F;
+    }
+
+    template <typename InputStream, typename OutputStream>
+    static bool Validate(InputStream& is, OutputStream& os) {
+        uint8_t c = static_cast<uint8_t>(is.Take());
+        os.Put(static_cast<typename OutputStream::Ch>(c));
+        return c <= 0x7F;
+    }
+
+    template <typename InputByteStream>
+    static CharType TakeBOM(InputByteStream& is) {
+        RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1);
+        uint8_t c = static_cast<uint8_t>(Take(is));
+        return static_cast<Ch>(c);
+    }
+
+    template <typename InputByteStream>
+    static Ch Take(InputByteStream& is) {
+        RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1);
+        return static_cast<Ch>(is.Take());
+    }
+
+    template <typename OutputByteStream>
+    static void PutBOM(OutputByteStream& os) {
+        RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1);
+        (void)os;
+    }
+
+    template <typename OutputByteStream>
+    static void Put(OutputByteStream& os, Ch c) {
+        RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1);
+        os.Put(static_cast<typename OutputByteStream::Ch>(c));
+    }
+};
+
+///////////////////////////////////////////////////////////////////////////////
+// AutoUTF
+
+//! Runtime-specified UTF encoding type of a stream.
+enum UTFType {
+    kUTF8 = 0,      //!< UTF-8.
+    kUTF16LE = 1,   //!< UTF-16 little endian.
+    kUTF16BE = 2,   //!< UTF-16 big endian.
+    kUTF32LE = 3,   //!< UTF-32 little endian.
+    kUTF32BE = 4    //!< UTF-32 big endian.
+};
+
+//! Dynamically select encoding according to stream's runtime-specified UTF encoding type.
+/*! \note This class can be used with AutoUTFInputtStream and AutoUTFOutputStream, which provides GetType().
+*/
+template<typename CharType>
+struct AutoUTF {
+    typedef CharType Ch;
+
+    enum { supportUnicode = 1 };
+
+#define RAPIDJSON_ENCODINGS_FUNC(x) UTF8<Ch>::x, UTF16LE<Ch>::x, UTF16BE<Ch>::x, UTF32LE<Ch>::x, UTF32BE<Ch>::x
+
+    template<typename OutputStream>
+    static RAPIDJSON_FORCEINLINE void Encode(OutputStream& os, unsigned codepoint) {
+        typedef void (*EncodeFunc)(OutputStream&, unsigned);
+        static const EncodeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Encode) };
+        (*f[os.GetType()])(os, codepoint);
+    }
+
+    template<typename OutputStream>
+    static RAPIDJSON_FORCEINLINE void EncodeUnsafe(OutputStream& os, unsigned codepoint) {
+        typedef void (*EncodeFunc)(OutputStream&, unsigned);
+        static const EncodeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(EncodeUnsafe) };
+        (*f[os.GetType()])(os, codepoint);
+    }
+
+    template <typename InputStream>
+    static RAPIDJSON_FORCEINLINE bool Decode(InputStream& is, unsigned* codepoint) {
+        typedef bool (*DecodeFunc)(InputStream&, unsigned*);
+        static const DecodeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Decode) };
+        return (*f[is.GetType()])(is, codepoint);
+    }
+
+    template <typename InputStream, typename OutputStream>
+    static RAPIDJSON_FORCEINLINE bool Validate(InputStream& is, OutputStream& os) {
+        typedef bool (*ValidateFunc)(InputStream&, OutputStream&);
+        static const ValidateFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Validate) };
+        return (*f[is.GetType()])(is, os);
+    }
+
+#undef RAPIDJSON_ENCODINGS_FUNC
+};
+
+///////////////////////////////////////////////////////////////////////////////
+// Transcoder
+
+//! Encoding conversion.
+template<typename SourceEncoding, typename TargetEncoding>
+struct Transcoder {
+    //! Take one Unicode codepoint from source encoding, convert it to target encoding and put it to the output stream.
+    template<typename InputStream, typename OutputStream>
+    static RAPIDJSON_FORCEINLINE bool Transcode(InputStream& is, OutputStream& os) {
+        unsigned codepoint;
+        if (!SourceEncoding::Decode(is, &codepoint))
+            return false;
+        TargetEncoding::Encode(os, codepoint);
+        return true;
+    }
+
+    template<typename InputStream, typename OutputStream>
+    static RAPIDJSON_FORCEINLINE bool TranscodeUnsafe(InputStream& is, OutputStream& os) {
+        unsigned codepoint;
+        if (!SourceEncoding::Decode(is, &codepoint))
+            return false;
+        TargetEncoding::EncodeUnsafe(os, codepoint);
+        return true;
+    }
+
+    //! Validate one Unicode codepoint from an encoded stream.
+    template<typename InputStream, typename OutputStream>
+    static RAPIDJSON_FORCEINLINE bool Validate(InputStream& is, OutputStream& os) {
+        return Transcode(is, os);   // Since source/target encoding is different, must transcode.
+    }
+};
+
+// Forward declaration.
+template<typename Stream>
+inline void PutUnsafe(Stream& stream, typename Stream::Ch c);
+
+//! Specialization of Transcoder with same source and target encoding.
+template<typename Encoding>
+struct Transcoder<Encoding, Encoding> {
+    template<typename InputStream, typename OutputStream>
+    static RAPIDJSON_FORCEINLINE bool Transcode(InputStream& is, OutputStream& os) {
+        os.Put(is.Take());  // Just copy one code unit. This semantic is different from primary template class.
+        return true;
+    }
+    
+    template<typename InputStream, typename OutputStream>
+    static RAPIDJSON_FORCEINLINE bool TranscodeUnsafe(InputStream& is, OutputStream& os) {
+        PutUnsafe(os, is.Take());  // Just copy one code unit. This semantic is different from primary template class.
+        return true;
+    }
+    
+    template<typename InputStream, typename OutputStream>
+    static RAPIDJSON_FORCEINLINE bool Validate(InputStream& is, OutputStream& os) {
+        return Encoding::Validate(is, os);  // source/target encoding are the same
+    }
+};
+
+RAPIDJSON_NAMESPACE_END
+
+#if defined(__GNUC__) || (defined(_MSC_VER) && !defined(__clang__))
+RAPIDJSON_DIAG_POP
+#endif
+
+#endif // RAPIDJSON_ENCODINGS_H_

vendor/github.com/Benau/go_rlottie/lottie_rapidjson_error_en.h

@@ -0,0 +1,74 @@
+// Tencent is pleased to support the open source community by making RapidJSON available.
+// 
+// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved.
+//
+// Licensed under the MIT License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// http://opensource.org/licenses/MIT
+//
+// Unless required by applicable law or agreed to in writing, software distributed 
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the 
+// specific language governing permissions and limitations under the License.
+
+#ifndef RAPIDJSON_ERROR_EN_H_
+#define RAPIDJSON_ERROR_EN_H_
+
+#include "lottie_rapidjson_error_error.h"
+
+#ifdef __clang__
+RAPIDJSON_DIAG_PUSH
+RAPIDJSON_DIAG_OFF(switch-enum)
+RAPIDJSON_DIAG_OFF(covered-switch-default)
+#endif
+
+RAPIDJSON_NAMESPACE_BEGIN
+
+//! Maps error code of parsing into error message.
+/*!
+    \ingroup RAPIDJSON_ERRORS
+    \param parseErrorCode Error code obtained in parsing.
+    \return the error message.
+    \note User can make a copy of this function for localization.
+        Using switch-case is safer for future modification of error codes.
+*/
+inline const RAPIDJSON_ERROR_CHARTYPE* GetParseError_En(ParseErrorCode parseErrorCode) {
+    switch (parseErrorCode) {
+        case kParseErrorNone:                           return RAPIDJSON_ERROR_STRING("No error.");
+
+        case kParseErrorDocumentEmpty:                  return RAPIDJSON_ERROR_STRING("The document is empty.");
+        case kParseErrorDocumentRootNotSingular:        return RAPIDJSON_ERROR_STRING("The document root must not be followed by other values.");
+    
+        case kParseErrorValueInvalid:                   return RAPIDJSON_ERROR_STRING("Invalid value.");
+    
+        case kParseErrorObjectMissName:                 return RAPIDJSON_ERROR_STRING("Missing a name for object member.");
+        case kParseErrorObjectMissColon:                return RAPIDJSON_ERROR_STRING("Missing a colon after a name of object member.");
+        case kParseErrorObjectMissCommaOrCurlyBracket:  return RAPIDJSON_ERROR_STRING("Missing a comma or '}' after an object member.");
+    
+        case kParseErrorArrayMissCommaOrSquareBracket:  return RAPIDJSON_ERROR_STRING("Missing a comma or ']' after an array element.");
+
+        case kParseErrorStringUnicodeEscapeInvalidHex:  return RAPIDJSON_ERROR_STRING("Incorrect hex digit after \\u escape in string.");
+        case kParseErrorStringUnicodeSurrogateInvalid:  return RAPIDJSON_ERROR_STRING("The surrogate pair in string is invalid.");
+        case kParseErrorStringEscapeInvalid:            return RAPIDJSON_ERROR_STRING("Invalid escape character in string.");
+        case kParseErrorStringMissQuotationMark:        return RAPIDJSON_ERROR_STRING("Missing a closing quotation mark in string.");
+        case kParseErrorStringInvalidEncoding:          return RAPIDJSON_ERROR_STRING("Invalid encoding in string.");
+
+        case kParseErrorNumberTooBig:                   return RAPIDJSON_ERROR_STRING("Number too big to be stored in double.");
+        case kParseErrorNumberMissFraction:             return RAPIDJSON_ERROR_STRING("Miss fraction part in number.");
+        case kParseErrorNumberMissExponent:             return RAPIDJSON_ERROR_STRING("Miss exponent in number.");
+
+        case kParseErrorTermination:                    return RAPIDJSON_ERROR_STRING("Terminate parsing due to Handler error.");
+        case kParseErrorUnspecificSyntaxError:          return RAPIDJSON_ERROR_STRING("Unspecific syntax error.");
+
+        default:                                        return RAPIDJSON_ERROR_STRING("Unknown error.");
+    }
+}
+
+RAPIDJSON_NAMESPACE_END
+
+#ifdef __clang__
+RAPIDJSON_DIAG_POP
+#endif
+
+#endif // RAPIDJSON_ERROR_EN_H_

vendor/github.com/Benau/go_rlottie/lottie_rapidjson_error_error.h

@@ -0,0 +1,161 @@
+// Tencent is pleased to support the open source community by making RapidJSON available.
+// 
+// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved.
+//
+// Licensed under the MIT License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// http://opensource.org/licenses/MIT
+//
+// Unless required by applicable law or agreed to in writing, software distributed 
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the 
+// specific language governing permissions and limitations under the License.
+
+#ifndef RAPIDJSON_ERROR_ERROR_H_
+#define RAPIDJSON_ERROR_ERROR_H_
+
+#include "lottie_rapidjson_rapidjson.h"
+
+#ifdef __clang__
+RAPIDJSON_DIAG_PUSH
+RAPIDJSON_DIAG_OFF(padded)
+#endif
+
+/*! \file error.h */
+
+/*! \defgroup RAPIDJSON_ERRORS RapidJSON error handling */
+
+///////////////////////////////////////////////////////////////////////////////
+// RAPIDJSON_ERROR_CHARTYPE
+
+//! Character type of error messages.
+/*! \ingroup RAPIDJSON_ERRORS
+    The default character type is \c char.
+    On Windows, user can define this macro as \c TCHAR for supporting both
+    unicode/non-unicode settings.
+*/
+#ifndef RAPIDJSON_ERROR_CHARTYPE
+#define RAPIDJSON_ERROR_CHARTYPE char
+#endif
+
+///////////////////////////////////////////////////////////////////////////////
+// RAPIDJSON_ERROR_STRING
+
+//! Macro for converting string literial to \ref RAPIDJSON_ERROR_CHARTYPE[].
+/*! \ingroup RAPIDJSON_ERRORS
+    By default this conversion macro does nothing.
+    On Windows, user can define this macro as \c _T(x) for supporting both
+    unicode/non-unicode settings.
+*/
+#ifndef RAPIDJSON_ERROR_STRING
+#define RAPIDJSON_ERROR_STRING(x) x
+#endif
+
+RAPIDJSON_NAMESPACE_BEGIN
+
+///////////////////////////////////////////////////////////////////////////////
+// ParseErrorCode
+
+//! Error code of parsing.
+/*! \ingroup RAPIDJSON_ERRORS
+    \see GenericReader::Parse, GenericReader::GetParseErrorCode
+*/
+enum ParseErrorCode {
+    kParseErrorNone = 0,                        //!< No error.
+
+    kParseErrorDocumentEmpty,                   //!< The document is empty.
+    kParseErrorDocumentRootNotSingular,         //!< The document root must not follow by other values.
+
+    kParseErrorValueInvalid,                    //!< Invalid value.
+
+    kParseErrorObjectMissName,                  //!< Missing a name for object member.
+    kParseErrorObjectMissColon,                 //!< Missing a colon after a name of object member.
+    kParseErrorObjectMissCommaOrCurlyBracket,   //!< Missing a comma or '}' after an object member.
+
+    kParseErrorArrayMissCommaOrSquareBracket,   //!< Missing a comma or ']' after an array element.
+
+    kParseErrorStringUnicodeEscapeInvalidHex,   //!< Incorrect hex digit after \\u escape in string.
+    kParseErrorStringUnicodeSurrogateInvalid,   //!< The surrogate pair in string is invalid.
+    kParseErrorStringEscapeInvalid,             //!< Invalid escape character in string.
+    kParseErrorStringMissQuotationMark,         //!< Missing a closing quotation mark in string.
+    kParseErrorStringInvalidEncoding,           //!< Invalid encoding in string.
+
+    kParseErrorNumberTooBig,                    //!< Number too big to be stored in double.
+    kParseErrorNumberMissFraction,              //!< Miss fraction part in number.
+    kParseErrorNumberMissExponent,              //!< Miss exponent in number.
+
+    kParseErrorTermination,                     //!< Parsing was terminated.
+    kParseErrorUnspecificSyntaxError            //!< Unspecific syntax error.
+};
+
+//! Result of parsing (wraps ParseErrorCode)
+/*!
+    \ingroup RAPIDJSON_ERRORS
+    \code
+        Document doc;
+        ParseResult ok = doc.Parse("[42]");
+        if (!ok) {
+            fprintf(stderr, "JSON parse error: %s (%u)",
+                    GetParseError_En(ok.Code()), ok.Offset());
+            exit(EXIT_FAILURE);
+        }
+    \endcode
+    \see GenericReader::Parse, GenericDocument::Parse
+*/
+struct ParseResult {
+    //!! Unspecified boolean type
+    typedef bool (ParseResult::*BooleanType)() const;
+public:
+    //! Default constructor, no error.
+    ParseResult() : code_(kParseErrorNone), offset_(0) {}
+    //! Constructor to set an error.
+    ParseResult(ParseErrorCode code, size_t offset) : code_(code), offset_(offset) {}
+
+    //! Get the error code.
+    ParseErrorCode Code() const { return code_; }
+    //! Get the error offset, if \ref IsError(), 0 otherwise.
+    size_t Offset() const { return offset_; }
+
+    //! Explicit conversion to \c bool, returns \c true, iff !\ref IsError().
+    operator BooleanType() const { return !IsError() ? &ParseResult::IsError : NULL; }
+    //! Whether the result is an error.
+    bool IsError() const { return code_ != kParseErrorNone; }
+
+    bool operator==(const ParseResult& that) const { return code_ == that.code_; }
+    bool operator==(ParseErrorCode code) const { return code_ == code; }
+    friend bool operator==(ParseErrorCode code, const ParseResult & err) { return code == err.code_; }
+
+    bool operator!=(const ParseResult& that) const { return !(*this == that); }
+    bool operator!=(ParseErrorCode code) const { return !(*this == code); }
+    friend bool operator!=(ParseErrorCode code, const ParseResult & err) { return err != code; }
+
+    //! Reset error code.
+    void Clear() { Set(kParseErrorNone); }
+    //! Update error code and offset.
+    void Set(ParseErrorCode code, size_t offset = 0) { code_ = code; offset_ = offset; }
+
+private:
+    ParseErrorCode code_;
+    size_t offset_;
+};
+
+//! Function pointer type of GetParseError().
+/*! \ingroup RAPIDJSON_ERRORS
+
+    This is the prototype for \c GetParseError_X(), where \c X is a locale.
+    User can dynamically change locale in runtime, e.g.:
+\code
+    GetParseErrorFunc GetParseError = GetParseError_En; // or whatever
+    const RAPIDJSON_ERROR_CHARTYPE* s = GetParseError(document.GetParseErrorCode());
+\endcode
+*/
+typedef const RAPIDJSON_ERROR_CHARTYPE* (*GetParseErrorFunc)(ParseErrorCode);
+
+RAPIDJSON_NAMESPACE_END
+
+#ifdef __clang__
+RAPIDJSON_DIAG_POP
+#endif
+
+#endif // RAPIDJSON_ERROR_ERROR_H_

vendor/github.com/Benau/go_rlottie/lottie_rapidjson_filereadstream.h

@@ -0,0 +1,99 @@
+// Tencent is pleased to support the open source community by making RapidJSON available.
+// 
+// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved.
+//
+// Licensed under the MIT License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// http://opensource.org/licenses/MIT
+//
+// Unless required by applicable law or agreed to in writing, software distributed 
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the 
+// specific language governing permissions and limitations under the License.
+
+#ifndef RAPIDJSON_FILEREADSTREAM_H_
+#define RAPIDJSON_FILEREADSTREAM_H_
+
+#include "lottie_rapidjson_stream.h"
+#include <cstdio>
+
+#ifdef __clang__
+RAPIDJSON_DIAG_PUSH
+RAPIDJSON_DIAG_OFF(padded)
+RAPIDJSON_DIAG_OFF(unreachable-code)
+RAPIDJSON_DIAG_OFF(missing-noreturn)
+#endif
+
+RAPIDJSON_NAMESPACE_BEGIN
+
+//! File byte stream for input using fread().
+/*!
+    \note implements Stream concept
+*/
+class FileReadStream {
+public:
+    typedef char Ch;    //!< Character type (byte).
+
+    //! Constructor.
+    /*!
+        \param fp File pointer opened for read.
+        \param buffer user-supplied buffer.
+        \param bufferSize size of buffer in bytes. Must >=4 bytes.
+    */
+    FileReadStream(std::FILE* fp, char* buffer, size_t bufferSize) : fp_(fp), buffer_(buffer), bufferSize_(bufferSize), bufferLast_(0), current_(buffer_), readCount_(0), count_(0), eof_(false) { 
+        RAPIDJSON_ASSERT(fp_ != 0);
+        RAPIDJSON_ASSERT(bufferSize >= 4);
+        Read();
+    }
+
+    Ch Peek() const { return *current_; }
+    Ch Take() { Ch c = *current_; Read(); return c; }
+    size_t Tell() const { return count_ + static_cast<size_t>(current_ - buffer_); }
+
+    // Not implemented
+    void Put(Ch) { RAPIDJSON_ASSERT(false); }
+    void Flush() { RAPIDJSON_ASSERT(false); } 
+    Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; }
+    size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; }
+
+    // For encoding detection only.
+    const Ch* Peek4() const {
+        return (current_ + 4 - !eof_ <= bufferLast_) ? current_ : 0;
+    }
+
+private:
+    void Read() {
+        if (current_ < bufferLast_)
+            ++current_;
+        else if (!eof_) {
+            count_ += readCount_;
+            readCount_ = std::fread(buffer_, 1, bufferSize_, fp_);
+            bufferLast_ = buffer_ + readCount_ - 1;
+            current_ = buffer_;
+
+            if (readCount_ < bufferSize_) {
+                buffer_[readCount_] = '\0';
+                ++bufferLast_;
+                eof_ = true;
+            }
+        }
+    }
+
+    std::FILE* fp_;
+    Ch *buffer_;
+    size_t bufferSize_;
+    Ch *bufferLast_;
+    Ch *current_;
+    size_t readCount_;
+    size_t count_;  //!< Number of characters read
+    bool eof_;
+};
+
+RAPIDJSON_NAMESPACE_END
+
+#ifdef __clang__
+RAPIDJSON_DIAG_POP
+#endif
+
+#endif // RAPIDJSON_FILESTREAM_H_

vendor/github.com/Benau/go_rlottie/lottie_rapidjson_filewritestream.h

@@ -0,0 +1,104 @@
+// Tencent is pleased to support the open source community by making RapidJSON available.
+// 
+// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved.
+//
+// Licensed under the MIT License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// http://opensource.org/licenses/MIT
+//
+// Unless required by applicable law or agreed to in writing, software distributed 
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the 
+// specific language governing permissions and limitations under the License.
+
+#ifndef RAPIDJSON_FILEWRITESTREAM_H_
+#define RAPIDJSON_FILEWRITESTREAM_H_
+
+#include "lottie_rapidjson_stream.h"
+#include <cstdio>
+
+#ifdef __clang__
+RAPIDJSON_DIAG_PUSH
+RAPIDJSON_DIAG_OFF(unreachable-code)
+#endif
+
+RAPIDJSON_NAMESPACE_BEGIN
+
+//! Wrapper of C file stream for output using fwrite().
+/*!
+    \note implements Stream concept
+*/
+class FileWriteStream {
+public:
+    typedef char Ch;    //!< Character type. Only support char.
+
+    FileWriteStream(std::FILE* fp, char* buffer, size_t bufferSize) : fp_(fp), buffer_(buffer), bufferEnd_(buffer + bufferSize), current_(buffer_) { 
+        RAPIDJSON_ASSERT(fp_ != 0);
+    }
+
+    void Put(char c) { 
+        if (current_ >= bufferEnd_)
+            Flush();
+
+        *current_++ = c;
+    }
+
+    void PutN(char c, size_t n) {
+        size_t avail = static_cast<size_t>(bufferEnd_ - current_);
+        while (n > avail) {
+            std::memset(current_, c, avail);
+            current_ += avail;
+            Flush();
+            n -= avail;
+            avail = static_cast<size_t>(bufferEnd_ - current_);
+        }
+
+        if (n > 0) {
+            std::memset(current_, c, n);
+            current_ += n;
+        }
+    }
+
+    void Flush() {
+        if (current_ != buffer_) {
+            size_t result = std::fwrite(buffer_, 1, static_cast<size_t>(current_ - buffer_), fp_);
+            if (result < static_cast<size_t>(current_ - buffer_)) {
+                // failure deliberately ignored at this time
+                // added to avoid warn_unused_result build errors
+            }
+            current_ = buffer_;
+        }
+    }
+
+    // Not implemented
+    char Peek() const { RAPIDJSON_ASSERT(false); return 0; }
+    char Take() { RAPIDJSON_ASSERT(false); return 0; }
+    size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; }
+    char* PutBegin() { RAPIDJSON_ASSERT(false); return 0; }
+    size_t PutEnd(char*) { RAPIDJSON_ASSERT(false); return 0; }
+
+private:
+    // Prohibit copy constructor & assignment operator.
+    FileWriteStream(const FileWriteStream&);
+    FileWriteStream& operator=(const FileWriteStream&);
+
+    std::FILE* fp_;
+    char *buffer_;
+    char *bufferEnd_;
+    char *current_;
+};
+
+//! Implement specialized version of PutN() with memset() for better performance.
+template<>
+inline void PutN(FileWriteStream& stream, char c, size_t n) {
+    stream.PutN(c, n);
+}
+
+RAPIDJSON_NAMESPACE_END
+
+#ifdef __clang__
+RAPIDJSON_DIAG_POP
+#endif
+
+#endif // RAPIDJSON_FILESTREAM_H_

vendor/github.com/Benau/go_rlottie/lottie_rapidjson_fwd.h

@@ -0,0 +1,151 @@
+// Tencent is pleased to support the open source community by making RapidJSON available.
+// 
+// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved.
+//
+// Licensed under the MIT License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// http://opensource.org/licenses/MIT
+//
+// Unless required by applicable law or agreed to in writing, software distributed 
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the 
+// specific language governing permissions and limitations under the License.
+
+#ifndef RAPIDJSON_FWD_H_
+#define RAPIDJSON_FWD_H_
+
+#include "lottie_rapidjson_rapidjson.h"
+
+RAPIDJSON_NAMESPACE_BEGIN
+
+// encodings.h
+
+template<typename CharType> struct UTF8;
+template<typename CharType> struct UTF16;
+template<typename CharType> struct UTF16BE;
+template<typename CharType> struct UTF16LE;
+template<typename CharType> struct UTF32;
+template<typename CharType> struct UTF32BE;
+template<typename CharType> struct UTF32LE;
+template<typename CharType> struct ASCII;
+template<typename CharType> struct AutoUTF;
+
+template<typename SourceEncoding, typename TargetEncoding>
+struct Transcoder;
+
+// allocators.h
+
+class CrtAllocator;
+
+template <typename BaseAllocator>
+class MemoryPoolAllocator;
+
+// stream.h
+
+template <typename Encoding>
+struct GenericStringStream;
+
+typedef GenericStringStream<UTF8<char> > StringStream;
+
+template <typename Encoding>
+struct GenericInsituStringStream;
+
+typedef GenericInsituStringStream<UTF8<char> > InsituStringStream;
+
+// stringbuffer.h
+
+template <typename Encoding, typename Allocator>
+class GenericStringBuffer;
+
+typedef GenericStringBuffer<UTF8<char>, CrtAllocator> StringBuffer;
+
+// filereadstream.h
+
+class FileReadStream;
+
+// filewritestream.h
+
+class FileWriteStream;
+
+// memorybuffer.h
+
+template <typename Allocator>
+struct GenericMemoryBuffer;
+
+typedef GenericMemoryBuffer<CrtAllocator> MemoryBuffer;
+
+// memorystream.h
+
+struct MemoryStream;
+
+// reader.h
+
+template<typename Encoding, typename Derived>
+struct BaseReaderHandler;
+
+template <typename SourceEncoding, typename TargetEncoding, typename StackAllocator>
+class GenericReader;
+
+typedef GenericReader<UTF8<char>, UTF8<char>, CrtAllocator> Reader;
+
+// writer.h
+
+template<typename OutputStream, typename SourceEncoding, typename TargetEncoding, typename StackAllocator, unsigned writeFlags>
+class Writer;
+
+// prettywriter.h
+
+template<typename OutputStream, typename SourceEncoding, typename TargetEncoding, typename StackAllocator, unsigned writeFlags>
+class PrettyWriter;
+
+// document.h
+
+template <typename Encoding, typename Allocator> 
+class GenericMember;
+
+template <bool Const, typename Encoding, typename Allocator>
+class GenericMemberIterator;
+
+template<typename CharType>
+struct GenericStringRef;
+
+template <typename Encoding, typename Allocator> 
+class GenericValue;
+
+typedef GenericValue<UTF8<char>, MemoryPoolAllocator<CrtAllocator> > Value;
+
+template <typename Encoding, typename Allocator, typename StackAllocator>
+class GenericDocument;
+
+typedef GenericDocument<UTF8<char>, MemoryPoolAllocator<CrtAllocator>, CrtAllocator> Document;
+
+// pointer.h
+
+template <typename ValueType, typename Allocator>
+class GenericPointer;
+
+typedef GenericPointer<Value, CrtAllocator> Pointer;
+
+// schema.h
+
+template <typename SchemaDocumentType>
+class IGenericRemoteSchemaDocumentProvider;
+
+template <typename ValueT, typename Allocator>
+class GenericSchemaDocument;
+
+typedef GenericSchemaDocument<Value, CrtAllocator> SchemaDocument;
+typedef IGenericRemoteSchemaDocumentProvider<SchemaDocument> IRemoteSchemaDocumentProvider;
+
+template <
+    typename SchemaDocumentType,
+    typename OutputHandler,
+    typename StateAllocator>
+class GenericSchemaValidator;
+
+typedef GenericSchemaValidator<SchemaDocument, BaseReaderHandler<UTF8<char>, void>, CrtAllocator> SchemaValidator;
+
+RAPIDJSON_NAMESPACE_END
+
+#endif // RAPIDJSON_RAPIDJSONFWD_H_

vendor/github.com/Benau/go_rlottie/lottie_rapidjson_internal_biginteger.h

@@ -0,0 +1,290 @@
+// Tencent is pleased to support the open source community by making RapidJSON available.
+// 
+// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved.
+//
+// Licensed under the MIT License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// http://opensource.org/licenses/MIT
+//
+// Unless required by applicable law or agreed to in writing, software distributed 
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the 
+// specific language governing permissions and limitations under the License.
+
+#ifndef RAPIDJSON_BIGINTEGER_H_
+#define RAPIDJSON_BIGINTEGER_H_
+
+#include "lottie_rapidjson_rapidjson.h"
+
+#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) && defined(_M_AMD64)
+#include <intrin.h> // for _umul128
+#pragma intrinsic(_umul128)
+#endif
+
+RAPIDJSON_NAMESPACE_BEGIN
+namespace internal {
+
+class BigInteger {
+public:
+    typedef uint64_t Type;
+
+    BigInteger(const BigInteger& rhs) : count_(rhs.count_) {
+        std::memcpy(digits_, rhs.digits_, count_ * sizeof(Type));
+    }
+
+    explicit BigInteger(uint64_t u) : count_(1) {
+        digits_[0] = u;
+    }
+
+    BigInteger(const char* decimals, size_t length) : count_(1) {
+        RAPIDJSON_ASSERT(length > 0);
+        digits_[0] = 0;
+        size_t i = 0;
+        const size_t kMaxDigitPerIteration = 19;  // 2^64 = 18446744073709551616 > 10^19
+        while (length >= kMaxDigitPerIteration) {
+            AppendDecimal64(decimals + i, decimals + i + kMaxDigitPerIteration);
+            length -= kMaxDigitPerIteration;
+            i += kMaxDigitPerIteration;
+        }
+
+        if (length > 0)
+            AppendDecimal64(decimals + i, decimals + i + length);
+    }
+    
+    BigInteger& operator=(const BigInteger &rhs)
+    {
+        if (this != &rhs) {
+            count_ = rhs.count_;
+            std::memcpy(digits_, rhs.digits_, count_ * sizeof(Type));
+        }
+        return *this;
+    }
+    
+    BigInteger& operator=(uint64_t u) {
+        digits_[0] = u;            
+        count_ = 1;
+        return *this;
+    }
+
+    BigInteger& operator+=(uint64_t u) {
+        Type backup = digits_[0];
+        digits_[0] += u;
+        for (size_t i = 0; i < count_ - 1; i++) {
+            if (digits_[i] >= backup)
+                return *this; // no carry
+            backup = digits_[i + 1];
+            digits_[i + 1] += 1;
+        }
+
+        // Last carry
+        if (digits_[count_ - 1] < backup)
+            PushBack(1);
+
+        return *this;
+    }
+
+    BigInteger& operator*=(uint64_t u) {
+        if (u == 0) return *this = 0;
+        if (u == 1) return *this;
+        if (*this == 1) return *this = u;
+
+        uint64_t k = 0;
+        for (size_t i = 0; i < count_; i++) {
+            uint64_t hi;
+            digits_[i] = MulAdd64(digits_[i], u, k, &hi);
+            k = hi;
+        }
+        
+        if (k > 0)
+            PushBack(k);
+
+        return *this;
+    }
+
+    BigInteger& operator*=(uint32_t u) {
+        if (u == 0) return *this = 0;
+        if (u == 1) return *this;
+        if (*this == 1) return *this = u;
+
+        uint64_t k = 0;
+        for (size_t i = 0; i < count_; i++) {
+            const uint64_t c = digits_[i] >> 32;
+            const uint64_t d = digits_[i] & 0xFFFFFFFF;
+            const uint64_t uc = u * c;
+            const uint64_t ud = u * d;
+            const uint64_t p0 = ud + k;
+            const uint64_t p1 = uc + (p0 >> 32);
+            digits_[i] = (p0 & 0xFFFFFFFF) | (p1 << 32);
+            k = p1 >> 32;
+        }
+        
+        if (k > 0)
+            PushBack(k);
+
+        return *this;
+    }
+
+    BigInteger& operator<<=(size_t shift) {
+        if (IsZero() || shift == 0) return *this;
+
+        size_t offset = shift / kTypeBit;
+        size_t interShift = shift % kTypeBit;
+        RAPIDJSON_ASSERT(count_ + offset <= kCapacity);
+
+        if (interShift == 0) {
+            std::memmove(digits_ + offset, digits_, count_ * sizeof(Type));
+            count_ += offset;
+        }
+        else {
+            digits_[count_] = 0;
+            for (size_t i = count_; i > 0; i--)
+                digits_[i + offset] = (digits_[i] << interShift) | (digits_[i - 1] >> (kTypeBit - interShift));
+            digits_[offset] = digits_[0] << interShift;
+            count_ += offset;
+            if (digits_[count_])
+                count_++;
+        }
+
+        std::memset(digits_, 0, offset * sizeof(Type));
+
+        return *this;
+    }
+
+    bool operator==(const BigInteger& rhs) const {
+        return count_ == rhs.count_ && std::memcmp(digits_, rhs.digits_, count_ * sizeof(Type)) == 0;
+    }
+
+    bool operator==(const Type rhs) const {
+        return count_ == 1 && digits_[0] == rhs;
+    }
+
+    BigInteger& MultiplyPow5(unsigned exp) {
+        static const uint32_t kPow5[12] = {
+            5,
+            5 * 5,
+            5 * 5 * 5,
+            5 * 5 * 5 * 5,
+            5 * 5 * 5 * 5 * 5,
+            5 * 5 * 5 * 5 * 5 * 5,
+            5 * 5 * 5 * 5 * 5 * 5 * 5,
+            5 * 5 * 5 * 5 * 5 * 5 * 5 * 5,
+            5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5,
+            5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5,
+            5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5,
+            5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5
+        };
+        if (exp == 0) return *this;
+        for (; exp >= 27; exp -= 27) *this *= RAPIDJSON_UINT64_C2(0X6765C793, 0XFA10079D); // 5^27
+        for (; exp >= 13; exp -= 13) *this *= static_cast<uint32_t>(1220703125u); // 5^13
+        if (exp > 0)                 *this *= kPow5[exp - 1];
+        return *this;
+    }
+
+    // Compute absolute difference of this and rhs.
+    // Assume this != rhs
+    bool Difference(const BigInteger& rhs, BigInteger* out) const {
+        int cmp = Compare(rhs);
+        RAPIDJSON_ASSERT(cmp != 0);
+        const BigInteger *a, *b;  // Makes a > b
+        bool ret;
+        if (cmp < 0) { a = &rhs; b = this; ret = true; }
+        else         { a = this; b = &rhs; ret = false; }
+
+        Type borrow = 0;
+        for (size_t i = 0; i < a->count_; i++) {
+            Type d = a->digits_[i] - borrow;
+            if (i < b->count_)
+                d -= b->digits_[i];
+            borrow = (d > a->digits_[i]) ? 1 : 0;
+            out->digits_[i] = d;
+            if (d != 0)
+                out->count_ = i + 1;
+        }
+
+        return ret;
+    }
+
+    int Compare(const BigInteger& rhs) const {
+        if (count_ != rhs.count_)
+            return count_ < rhs.count_ ? -1 : 1;
+
+        for (size_t i = count_; i-- > 0;)
+            if (digits_[i] != rhs.digits_[i])
+                return digits_[i] < rhs.digits_[i] ? -1 : 1;
+
+        return 0;
+    }
+
+    size_t GetCount() const { return count_; }
+    Type GetDigit(size_t index) const { RAPIDJSON_ASSERT(index < count_); return digits_[index]; }
+    bool IsZero() const { return count_ == 1 && digits_[0] == 0; }
+
+private:
+    void AppendDecimal64(const char* begin, const char* end) {
+        uint64_t u = ParseUint64(begin, end);
+        if (IsZero())
+            *this = u;
+        else {
+            unsigned exp = static_cast<unsigned>(end - begin);
+            (MultiplyPow5(exp) <<= exp) += u;   // *this = *this * 10^exp + u
+        }
+    }
+
+    void PushBack(Type digit) {
+        RAPIDJSON_ASSERT(count_ < kCapacity);
+        digits_[count_++] = digit;
+    }
+
+    static uint64_t ParseUint64(const char* begin, const char* end) {
+        uint64_t r = 0;
+        for (const char* p = begin; p != end; ++p) {
+            RAPIDJSON_ASSERT(*p >= '0' && *p <= '9');
+            r = r * 10u + static_cast<unsigned>(*p - '0');
+        }
+        return r;
+    }
+
+    // Assume a * b + k < 2^128
+    static uint64_t MulAdd64(uint64_t a, uint64_t b, uint64_t k, uint64_t* outHigh) {
+#if defined(_MSC_VER) && defined(_M_AMD64)
+        uint64_t low = _umul128(a, b, outHigh) + k;
+        if (low < k)
+            (*outHigh)++;
+        return low;
+#elif (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) && defined(__x86_64__)
+        __extension__ typedef unsigned __int128 uint128;
+        uint128 p = static_cast<uint128>(a) * static_cast<uint128>(b);
+        p += k;
+        *outHigh = static_cast<uint64_t>(p >> 64);
+        return static_cast<uint64_t>(p);
+#else
+        const uint64_t a0 = a & 0xFFFFFFFF, a1 = a >> 32, b0 = b & 0xFFFFFFFF, b1 = b >> 32;
+        uint64_t x0 = a0 * b0, x1 = a0 * b1, x2 = a1 * b0, x3 = a1 * b1;
+        x1 += (x0 >> 32); // can't give carry
+        x1 += x2;
+        if (x1 < x2)
+            x3 += (static_cast<uint64_t>(1) << 32);
+        uint64_t lo = (x1 << 32) + (x0 & 0xFFFFFFFF);
+        uint64_t hi = x3 + (x1 >> 32);
+
+        lo += k;
+        if (lo < k)
+            hi++;
+        *outHigh = hi;
+        return lo;
+#endif
+    }
+
+    static const size_t kBitCount = 3328;  // 64bit * 54 > 10^1000
+    static const size_t kCapacity = kBitCount / sizeof(Type);
+    static const size_t kTypeBit = sizeof(Type) * 8;
+
+    Type digits_[kCapacity];
+    size_t count_;
+};
+
+} // namespace internal
+RAPIDJSON_NAMESPACE_END
+
+#endif // RAPIDJSON_BIGINTEGER_H_

vendor/github.com/Benau/go_rlottie/lottie_rapidjson_internal_clzll.h

@@ -0,0 +1,71 @@
+// Tencent is pleased to support the open source community by making RapidJSON available.
+//
+// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved.
+//
+// Licensed under the MIT License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// http://opensource.org/licenses/MIT
+//
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+#ifndef RAPIDJSON_CLZLL_H_
+#define RAPIDJSON_CLZLL_H_
+
+#include "lottie_rapidjson_rapidjson.h"
+
+#if defined(_MSC_VER) && !defined(UNDER_CE)
+#include <intrin.h>
+#if defined(_WIN64)
+#pragma intrinsic(_BitScanReverse64)
+#else
+#pragma intrinsic(_BitScanReverse)
+#endif
+#endif
+
+RAPIDJSON_NAMESPACE_BEGIN
+namespace internal {
+
+inline uint32_t clzll(uint64_t x) {
+    // Passing 0 to __builtin_clzll is UB in GCC and results in an
+    // infinite loop in the software implementation.
+    RAPIDJSON_ASSERT(x != 0);
+
+#if defined(_MSC_VER) && !defined(UNDER_CE)
+    unsigned long r = 0;
+#if defined(_WIN64)
+    _BitScanReverse64(&r, x);
+#else
+    // Scan the high 32 bits.
+    if (_BitScanReverse(&r, static_cast<uint32_t>(x >> 32)))
+        return 63 - (r + 32);
+
+    // Scan the low 32 bits.
+    _BitScanReverse(&r, static_cast<uint32_t>(x & 0xFFFFFFFF));
+#endif // _WIN64
+
+    return 63 - r;
+#elif (defined(__GNUC__) && __GNUC__ >= 4) || RAPIDJSON_HAS_BUILTIN(__builtin_clzll)
+    // __builtin_clzll wrapper
+    return static_cast<uint32_t>(__builtin_clzll(x));
+#else
+    // naive version
+    uint32_t r = 0;
+    while (!(x & (static_cast<uint64_t>(1) << 63))) {
+        x <<= 1;
+        ++r;
+    }
+
+    return r;
+#endif // _MSC_VER
+}
+
+#define RAPIDJSON_CLZLL RAPIDJSON_NAMESPACE::internal::clzll
+
+} // namespace internal
+RAPIDJSON_NAMESPACE_END
+
+#endif // RAPIDJSON_CLZLL_H_

vendor/github.com/Benau/go_rlottie/lottie_rapidjson_internal_diyfp.h

@@ -0,0 +1,257 @@
+// Tencent is pleased to support the open source community by making RapidJSON available.
+//
+// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved.
+//
+// Licensed under the MIT License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// http://opensource.org/licenses/MIT
+//
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+// This is a C++ header-only implementation of Grisu2 algorithm from the publication:
+// Loitsch, Florian. "Printing floating-point numbers quickly and accurately with
+// integers." ACM Sigplan Notices 45.6 (2010): 233-243.
+
+#ifndef RAPIDJSON_DIYFP_H_
+#define RAPIDJSON_DIYFP_H_
+
+#include "lottie_rapidjson_rapidjson.h"
+#include "lottie_rapidjson_internal_clzll.h"
+#include <limits>
+
+#if defined(_MSC_VER) && defined(_M_AMD64) && !defined(__INTEL_COMPILER)
+#include <intrin.h>
+#pragma intrinsic(_umul128)
+#endif
+
+RAPIDJSON_NAMESPACE_BEGIN
+namespace internal {
+
+#ifdef __GNUC__
+RAPIDJSON_DIAG_PUSH
+RAPIDJSON_DIAG_OFF(effc++)
+#endif
+
+#ifdef __clang__
+RAPIDJSON_DIAG_PUSH
+RAPIDJSON_DIAG_OFF(padded)
+#endif
+
+struct DiyFp {
+    DiyFp() : f(), e() {}
+
+    DiyFp(uint64_t fp, int exp) : f(fp), e(exp) {}
+
+    explicit DiyFp(double d) {
+        union {
+            double d;
+            uint64_t u64;
+        } u = { d };
+
+        int biased_e = static_cast<int>((u.u64 & kDpExponentMask) >> kDpSignificandSize);
+        uint64_t significand = (u.u64 & kDpSignificandMask);
+        if (biased_e != 0) {
+            f = significand + kDpHiddenBit;
+            e = biased_e - kDpExponentBias;
+        }
+        else {
+            f = significand;
+            e = kDpMinExponent + 1;
+        }
+    }
+
+    DiyFp operator-(const DiyFp& rhs) const {
+        return DiyFp(f - rhs.f, e);
+    }
+
+    DiyFp operator*(const DiyFp& rhs) const {
+#if defined(_MSC_VER) && defined(_M_AMD64)
+        uint64_t h;
+        uint64_t l = _umul128(f, rhs.f, &h);
+        if (l & (uint64_t(1) << 63)) // rounding
+            h++;
+        return DiyFp(h, e + rhs.e + 64);
+#elif (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) && defined(__x86_64__)
+        __extension__ typedef unsigned __int128 uint128;
+        uint128 p = static_cast<uint128>(f) * static_cast<uint128>(rhs.f);
+        uint64_t h = static_cast<uint64_t>(p >> 64);
+        uint64_t l = static_cast<uint64_t>(p);
+        if (l & (uint64_t(1) << 63)) // rounding
+            h++;
+        return DiyFp(h, e + rhs.e + 64);
+#else
+        const uint64_t M32 = 0xFFFFFFFF;
+        const uint64_t a = f >> 32;
+        const uint64_t b = f & M32;
+        const uint64_t c = rhs.f >> 32;
+        const uint64_t d = rhs.f & M32;
+        const uint64_t ac = a * c;
+        const uint64_t bc = b * c;
+        const uint64_t ad = a * d;
+        const uint64_t bd = b * d;
+        uint64_t tmp = (bd >> 32) + (ad & M32) + (bc & M32);
+        tmp += 1U << 31;  /// mult_round
+        return DiyFp(ac + (ad >> 32) + (bc >> 32) + (tmp >> 32), e + rhs.e + 64);
+#endif
+    }
+
+    DiyFp Normalize() const {
+        int s = static_cast<int>(clzll(f));
+        return DiyFp(f << s, e - s);
+    }
+
+    DiyFp NormalizeBoundary() const {
+        DiyFp res = *this;
+        while (!(res.f & (kDpHiddenBit << 1))) {
+            res.f <<= 1;
+            res.e--;
+        }
+        res.f <<= (kDiySignificandSize - kDpSignificandSize - 2);
+        res.e = res.e - (kDiySignificandSize - kDpSignificandSize - 2);
+        return res;
+    }
+
+    void NormalizedBoundaries(DiyFp* minus, DiyFp* plus) const {
+        DiyFp pl = DiyFp((f << 1) + 1, e - 1).NormalizeBoundary();
+        DiyFp mi = (f == kDpHiddenBit) ? DiyFp((f << 2) - 1, e - 2) : DiyFp((f << 1) - 1, e - 1);
+        mi.f <<= mi.e - pl.e;
+        mi.e = pl.e;
+        *plus = pl;
+        *minus = mi;
+    }
+
+    double ToDouble() const {
+        union {
+            double d;
+            uint64_t u64;
+        }u;
+        RAPIDJSON_ASSERT(f <= kDpHiddenBit + kDpSignificandMask);
+        if (e < kDpDenormalExponent) {
+            // Underflow.
+            return 0.0;
+        }
+        if (e >= kDpMaxExponent) {
+            // Overflow.
+            return std::numeric_limits<double>::infinity();
+        }
+        const uint64_t be = (e == kDpDenormalExponent && (f & kDpHiddenBit) == 0) ? 0 :
+            static_cast<uint64_t>(e + kDpExponentBias);
+        u.u64 = (f & kDpSignificandMask) | (be << kDpSignificandSize);
+        return u.d;
+    }
+
+    static const int kDiySignificandSize = 64;
+    static const int kDpSignificandSize = 52;
+    static const int kDpExponentBias = 0x3FF + kDpSignificandSize;
+    static const int kDpMaxExponent = 0x7FF - kDpExponentBias;
+    static const int kDpMinExponent = -kDpExponentBias;
+    static const int kDpDenormalExponent = -kDpExponentBias + 1;
+    static const uint64_t kDpExponentMask = RAPIDJSON_UINT64_C2(0x7FF00000, 0x00000000);
+    static const uint64_t kDpSignificandMask = RAPIDJSON_UINT64_C2(0x000FFFFF, 0xFFFFFFFF);
+    static const uint64_t kDpHiddenBit = RAPIDJSON_UINT64_C2(0x00100000, 0x00000000);
+
+    uint64_t f;
+    int e;
+};
+
+inline DiyFp GetCachedPowerByIndex(size_t index) {
+    // 10^-348, 10^-340, ..., 10^340
+    static const uint64_t kCachedPowers_F[] = {
+        RAPIDJSON_UINT64_C2(0xfa8fd5a0, 0x081c0288), RAPIDJSON_UINT64_C2(0xbaaee17f, 0xa23ebf76),
+        RAPIDJSON_UINT64_C2(0x8b16fb20, 0x3055ac76), RAPIDJSON_UINT64_C2(0xcf42894a, 0x5dce35ea),
+        RAPIDJSON_UINT64_C2(0x9a6bb0aa, 0x55653b2d), RAPIDJSON_UINT64_C2(0xe61acf03, 0x3d1a45df),
+        RAPIDJSON_UINT64_C2(0xab70fe17, 0xc79ac6ca), RAPIDJSON_UINT64_C2(0xff77b1fc, 0xbebcdc4f),
+        RAPIDJSON_UINT64_C2(0xbe5691ef, 0x416bd60c), RAPIDJSON_UINT64_C2(0x8dd01fad, 0x907ffc3c),
+        RAPIDJSON_UINT64_C2(0xd3515c28, 0x31559a83), RAPIDJSON_UINT64_C2(0x9d71ac8f, 0xada6c9b5),
+        RAPIDJSON_UINT64_C2(0xea9c2277, 0x23ee8bcb), RAPIDJSON_UINT64_C2(0xaecc4991, 0x4078536d),
+        RAPIDJSON_UINT64_C2(0x823c1279, 0x5db6ce57), RAPIDJSON_UINT64_C2(0xc2109436, 0x4dfb5637),
+        RAPIDJSON_UINT64_C2(0x9096ea6f, 0x3848984f), RAPIDJSON_UINT64_C2(0xd77485cb, 0x25823ac7),
+        RAPIDJSON_UINT64_C2(0xa086cfcd, 0x97bf97f4), RAPIDJSON_UINT64_C2(0xef340a98, 0x172aace5),
+        RAPIDJSON_UINT64_C2(0xb23867fb, 0x2a35b28e), RAPIDJSON_UINT64_C2(0x84c8d4df, 0xd2c63f3b),
+        RAPIDJSON_UINT64_C2(0xc5dd4427, 0x1ad3cdba), RAPIDJSON_UINT64_C2(0x936b9fce, 0xbb25c996),
+        RAPIDJSON_UINT64_C2(0xdbac6c24, 0x7d62a584), RAPIDJSON_UINT64_C2(0xa3ab6658, 0x0d5fdaf6),
+        RAPIDJSON_UINT64_C2(0xf3e2f893, 0xdec3f126), RAPIDJSON_UINT64_C2(0xb5b5ada8, 0xaaff80b8),
+        RAPIDJSON_UINT64_C2(0x87625f05, 0x6c7c4a8b), RAPIDJSON_UINT64_C2(0xc9bcff60, 0x34c13053),
+        RAPIDJSON_UINT64_C2(0x964e858c, 0x91ba2655), RAPIDJSON_UINT64_C2(0xdff97724, 0x70297ebd),
+        RAPIDJSON_UINT64_C2(0xa6dfbd9f, 0xb8e5b88f), RAPIDJSON_UINT64_C2(0xf8a95fcf, 0x88747d94),
+        RAPIDJSON_UINT64_C2(0xb9447093, 0x8fa89bcf), RAPIDJSON_UINT64_C2(0x8a08f0f8, 0xbf0f156b),
+        RAPIDJSON_UINT64_C2(0xcdb02555, 0x653131b6), RAPIDJSON_UINT64_C2(0x993fe2c6, 0xd07b7fac),
+        RAPIDJSON_UINT64_C2(0xe45c10c4, 0x2a2b3b06), RAPIDJSON_UINT64_C2(0xaa242499, 0x697392d3),
+        RAPIDJSON_UINT64_C2(0xfd87b5f2, 0x8300ca0e), RAPIDJSON_UINT64_C2(0xbce50864, 0x92111aeb),
+        RAPIDJSON_UINT64_C2(0x8cbccc09, 0x6f5088cc), RAPIDJSON_UINT64_C2(0xd1b71758, 0xe219652c),
+        RAPIDJSON_UINT64_C2(0x9c400000, 0x00000000), RAPIDJSON_UINT64_C2(0xe8d4a510, 0x00000000),
+        RAPIDJSON_UINT64_C2(0xad78ebc5, 0xac620000), RAPIDJSON_UINT64_C2(0x813f3978, 0xf8940984),
+        RAPIDJSON_UINT64_C2(0xc097ce7b, 0xc90715b3), RAPIDJSON_UINT64_C2(0x8f7e32ce, 0x7bea5c70),
+        RAPIDJSON_UINT64_C2(0xd5d238a4, 0xabe98068), RAPIDJSON_UINT64_C2(0x9f4f2726, 0x179a2245),
+        RAPIDJSON_UINT64_C2(0xed63a231, 0xd4c4fb27), RAPIDJSON_UINT64_C2(0xb0de6538, 0x8cc8ada8),
+        RAPIDJSON_UINT64_C2(0x83c7088e, 0x1aab65db), RAPIDJSON_UINT64_C2(0xc45d1df9, 0x42711d9a),
+        RAPIDJSON_UINT64_C2(0x924d692c, 0xa61be758), RAPIDJSON_UINT64_C2(0xda01ee64, 0x1a708dea),
+        RAPIDJSON_UINT64_C2(0xa26da399, 0x9aef774a), RAPIDJSON_UINT64_C2(0xf209787b, 0xb47d6b85),
+        RAPIDJSON_UINT64_C2(0xb454e4a1, 0x79dd1877), RAPIDJSON_UINT64_C2(0x865b8692, 0x5b9bc5c2),
+        RAPIDJSON_UINT64_C2(0xc83553c5, 0xc8965d3d), RAPIDJSON_UINT64_C2(0x952ab45c, 0xfa97a0b3),
+        RAPIDJSON_UINT64_C2(0xde469fbd, 0x99a05fe3), RAPIDJSON_UINT64_C2(0xa59bc234, 0xdb398c25),
+        RAPIDJSON_UINT64_C2(0xf6c69a72, 0xa3989f5c), RAPIDJSON_UINT64_C2(0xb7dcbf53, 0x54e9bece),
+        RAPIDJSON_UINT64_C2(0x88fcf317, 0xf22241e2), RAPIDJSON_UINT64_C2(0xcc20ce9b, 0xd35c78a5),
+        RAPIDJSON_UINT64_C2(0x98165af3, 0x7b2153df), RAPIDJSON_UINT64_C2(0xe2a0b5dc, 0x971f303a),
+        RAPIDJSON_UINT64_C2(0xa8d9d153, 0x5ce3b396), RAPIDJSON_UINT64_C2(0xfb9b7cd9, 0xa4a7443c),
+        RAPIDJSON_UINT64_C2(0xbb764c4c, 0xa7a44410), RAPIDJSON_UINT64_C2(0x8bab8eef, 0xb6409c1a),
+        RAPIDJSON_UINT64_C2(0xd01fef10, 0xa657842c), RAPIDJSON_UINT64_C2(0x9b10a4e5, 0xe9913129),
+        RAPIDJSON_UINT64_C2(0xe7109bfb, 0xa19c0c9d), RAPIDJSON_UINT64_C2(0xac2820d9, 0x623bf429),
+        RAPIDJSON_UINT64_C2(0x80444b5e, 0x7aa7cf85), RAPIDJSON_UINT64_C2(0xbf21e440, 0x03acdd2d),
+        RAPIDJSON_UINT64_C2(0x8e679c2f, 0x5e44ff8f), RAPIDJSON_UINT64_C2(0xd433179d, 0x9c8cb841),
+        RAPIDJSON_UINT64_C2(0x9e19db92, 0xb4e31ba9), RAPIDJSON_UINT64_C2(0xeb96bf6e, 0xbadf77d9),
+        RAPIDJSON_UINT64_C2(0xaf87023b, 0x9bf0ee6b)
+    };
+    static const int16_t kCachedPowers_E[] = {
+        -1220, -1193, -1166, -1140, -1113, -1087, -1060, -1034, -1007,  -980,
+        -954,  -927,  -901,  -874,  -847,  -821,  -794,  -768,  -741,  -715,
+        -688,  -661,  -635,  -608,  -582,  -555,  -529,  -502,  -475,  -449,
+        -422,  -396,  -369,  -343,  -316,  -289,  -263,  -236,  -210,  -183,
+        -157,  -130,  -103,   -77,   -50,   -24,     3,    30,    56,    83,
+        109,   136,   162,   189,   216,   242,   269,   295,   322,   348,
+        375,   402,   428,   455,   481,   508,   534,   561,   588,   614,
+        641,   667,   694,   720,   747,   774,   800,   827,   853,   880,
+        907,   933,   960,   986,  1013,  1039,  1066
+    };
+    RAPIDJSON_ASSERT(index < 87);
+    return DiyFp(kCachedPowers_F[index], kCachedPowers_E[index]);
+}
+
+inline DiyFp GetCachedPower(int e, int* K) {
+
+    //int k = static_cast<int>(ceil((-61 - e) * 0.30102999566398114)) + 374;
+    double dk = (-61 - e) * 0.30102999566398114 + 347;  // dk must be positive, so can do ceiling in positive
+    int k = static_cast<int>(dk);
+    if (dk - k > 0.0)
+        k++;
+
+    unsigned index = static_cast<unsigned>((k >> 3) + 1);
+    *K = -(-348 + static_cast<int>(index << 3));    // decimal exponent no need lookup table
+
+    return GetCachedPowerByIndex(index);
+}
+
+inline DiyFp GetCachedPower10(int exp, int *outExp) {
+    RAPIDJSON_ASSERT(exp >= -348);
+    unsigned index = static_cast<unsigned>(exp + 348) / 8u;
+    *outExp = -348 + static_cast<int>(index) * 8;
+    return GetCachedPowerByIndex(index);
+}
+
+#ifdef __GNUC__
+RAPIDJSON_DIAG_POP
+#endif
+
+#ifdef __clang__
+RAPIDJSON_DIAG_POP
+RAPIDJSON_DIAG_OFF(padded)
+#endif
+
+} // namespace internal
+RAPIDJSON_NAMESPACE_END
+
+#endif // RAPIDJSON_DIYFP_H_