Release v1.18.3 (#1229)

Signed-off-by: Tilo Spannagel <development@tilosp.de>

.dockerignore

@@ -0,0 +1,2 @@
+Dockerfile
+tgs.Dockerfile

.fixmie.yml

@@ -0,0 +1,3 @@
+go:
+  comments:
+    disabled: true

.github/ISSUE_TEMPLATE/Bug_report.md

@@ -1,6 +1,7 @@
 ---
 name: Bug report
 about: Create a report to help us improve. (Check the FAQ on the wiki first)
+labels: bug
 
 ---
 

.github/ISSUE_TEMPLATE/Feature_request.md

@@ -1,6 +1,7 @@
 ---
 name: Feature request
 about: Suggest an idea for this project
+labels: enhancement
 
 ---
 

.github/workflows/development.yml

@@ -0,0 +1,57 @@
+name: Development
+on: [push, pull_request]
+jobs:
+  lint:
+    name: golangci-lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 20
+      - name: Run golangci-lint
+        uses: golangci/golangci-lint-action@v2
+        with:
+          version: v1.29
+          args: "-v --new-from-rev HEAD~5"
+  test-build-upload:
+    strategy:
+      matrix:
+        go-version: [1.14.x, 1.15.x]
+        platform: [ubuntu-latest]
+    runs-on: ${{ matrix.platform }}
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+    - name: Checkout code
+      uses: actions/checkout@v2
+      with:
+          fetch-depth: 0
+    - name: Test
+      run: go test ./... -mod=vendor
+    - name: Build
+      run: |
+        mkdir -p output/{win,lin,arm,mac}
+        VERSION=$(git describe --tags)
+        GOOS=linux GOARCH=amd64 go build -mod=vendor -ldflags "-s -X main.githash=$(git log --pretty=format:'%h' -n 1)" -o output/lin/matterbridge-$VERSION-linux-amd64
+        GOOS=windows GOARCH=amd64 go build -mod=vendor -ldflags "-s -X main.githash=$(git log --pretty=format:'%h' -n 1)" -o output/win/matterbridge-$VERSION-windows-amd64.exe
+        GOOS=darwin GOARCH=amd64 go build -mod=vendor -ldflags "-s -X main.githash=$(git log --pretty=format:'%h' -n 1)" -o output/mac/matterbridge-$VERSION-darwin-amd64
+    - name: Upload linux 64-bit
+      if: startsWith(matrix.go-version,'1.15')
+      uses: actions/upload-artifact@v2
+      with:
+        name: matterbridge-linux-64bit
+        path: output/lin
+    - name: Upload windows 64-bit
+      if: startsWith(matrix.go-version,'1.15')
+      uses: actions/upload-artifact@v2
+      with:
+        name: matterbridge-windows-64bit
+        path: output/win
+    - name: Upload darwin 64-bit
+      if: startsWith(matrix.go-version,'1.15')
+      uses: actions/upload-artifact@v2
+      with:
+        name: matterbridge-darwin-64bit
+        path: output/mac

.gitignore

@@ -0,0 +1,6 @@
+# Exclude matterbridge binary
+/matterbridge
+/matterbridge.exe
+
+# Exclude configuration file
+matterbridge.toml

.golangci.yaml

@@ -0,0 +1,216 @@
+# For full documentation of the configuration options please
+# see: https://github.com/golangci/golangci-lint#config-file.
+
+# options for analysis running
+run:
+  # default concurrency is the available CPU number
+  # concurrency: 4
+
+  # timeout for analysis, e.g. 30s, 5m, default is 1m
+  deadline: 2m
+
+  # exit code when at least one issue was found, default is 1
+  issues-exit-code: 1
+
+  # include test files or not, default is true
+  tests: true
+
+  # list of build tags, all linters use it. Default is empty list.
+  build-tags:
+
+  # which dirs to skip: they won't be analyzed;
+  # can use regexp here: generated.*, regexp is applied on full path;
+  # default value is empty list, but next dirs are always skipped independently
+  # from this option's value:
+  #   	vendor$, third_party$, testdata$, examples$, Godeps$, builtin$
+  skip-dirs: gateway/bridgemap$
+
+  # which files to skip: they will be analyzed, but issues from them
+  # won't be reported. Default value is empty list, but there is
+  # no need to include all autogenerated files, we confidently recognize
+  # autogenerated files. If it's not please let us know.
+  skip-files:
+
+
+# output configuration options
+output:
+  # colored-line-number|line-number|json|tab|checkstyle, default is "colored-line-number"
+  format: colored-line-number
+
+  # print lines of code with issue, default is true
+  print-issued-lines: true
+
+  # print linter name in the end of issue text, default is true
+  print-linter-name: true
+
+
+# all available settings of specific linters, we can set an option for
+# a given linter even if we deactivate that same linter at runtime
+linters-settings:
+  errcheck:
+    # report about not checking of errors in type assertions: `a := b.(MyStruct)`;
+    # default is false: such cases aren't reported by default.
+    check-type-assertions: false
+
+    # report about assignment of errors to blank identifier: `num, _ := strconv.Atoi(numStr)`;
+    # default is false: such cases aren't reported by default.
+    check-blank: false
+  govet:
+    # report about shadowed variables
+    check-shadowing: true
+  golint:
+    # minimal confidence for issues, default is 0.8
+    min-confidence: 0.8
+  gofmt:
+    # simplify code: gofmt with `-s` option, true by default
+    simplify: true
+  goimports:
+    # put imports beginning with prefix after 3rd-party packages;
+    # it's a comma-separated list of prefixes
+    local-prefixes: github.com
+  gocyclo:
+    # minimal code complexity to report, 30 by default (but we recommend 10-20)
+    min-complexity: 15
+  maligned:
+    # print struct with more effective memory layout or not, false by default
+    suggest-new: true
+  dupl:
+    # tokens count to trigger issue, 150 by default
+    threshold: 150
+  goconst:
+    # minimal length of string constant, 3 by default
+    min-len: 3
+    # minimal occurrences count to trigger, 3 by default
+    min-occurrences: 3
+  depguard:
+    list-type: blacklist
+    include-go-root: false
+    packages:
+    # List of packages that we would want to blacklist for... reasons.
+  misspell:
+    # Correct spellings using locale preferences for US or UK.
+    # Default is to use a neutral variety of English.
+    # Setting locale to US will correct the British spelling of 'colour' to 'color'.
+    locale: US
+  lll:
+    # max line length, lines longer will be reported. Default is 120.
+    # '\t' is counted as 1 character by default, and can be changed with the tab-width option
+    line-length: 150
+    # tab width in spaces. Default to 1.
+    tab-width: 1
+  unused:
+    # treat code as a program (not a library) and report unused exported identifiers; default is false.
+    # XXX: if you enable this setting, unused will report a lot of false-positives in text editors:
+    # if it's called for subdir of a project it can't find funcs usages. All text editor integrations
+    # with golangci-lint call it on a directory with the changed file.
+    check-exported: false
+  unparam:
+    # Inspect exported functions, default is false. Set to true if no external program/library imports your code.
+    # XXX: if you enable this setting, unparam will report a lot of false-positives in text editors:
+    # if it's called for subdir of a project it can't find external interfaces. All text editor integrations
+    # with golangci-lint call it on a directory with the changed file.
+    check-exported: false
+  nakedret:
+    # make an issue if func has more lines of code than this setting and it has naked returns; default is 30
+    max-func-lines: 0 # Warn on all naked returns.
+  prealloc:
+    # XXX: we don't recommend using this linter before doing performance profiling.
+    # For most programs usage of prealloc will be a premature optimization.
+
+    # Report preallocation suggestions only on simple loops that have no returns/breaks/continues/gotos in them.
+    # True by default.
+    simple: true
+    range-loops: true # Report preallocation suggestions on range loops, true by default
+    for-loops: false # Report preallocation suggestions on for loops, false by default
+  gocritic:
+    # which checks should be enabled; can't be combined with 'disabled-checks';
+    # default are: [appendAssign assignOp caseOrder dupArg dupBranchBody dupCase flagDeref
+    # ifElseChain regexpMust singleCaseSwitch sloppyLen switchTrue typeSwitchVar underef
+    # unlambda unslice rangeValCopy defaultCaseOrder];
+    # all checks list: https://github.com/go-critic/checkers
+    # disabled for now - hugeParam
+    enabled-checks:
+      - appendAssign
+      - assignOp
+      - boolExprSimplify
+      - builtinShadow
+      - captLocal
+      - caseOrder
+      - commentedOutImport
+      - defaultCaseOrder
+      - dupArg
+      - dupBranchBody
+      - dupCase
+      - dupSubExpr
+      - elseif
+      - emptyFallthrough
+      - ifElseChain
+      - importShadow
+      - indexAlloc
+      - methodExprCall
+      - nestingReduce
+      - offBy1
+      - ptrToRefParam
+      - regexpMust
+      - singleCaseSwitch
+      - sloppyLen
+      - switchTrue
+      - typeSwitchVar
+      - typeUnparen
+      - underef
+      - unlambda
+      - unnecessaryBlock
+      - unslice
+      - valSwap
+      - wrapperFunc
+      - yodaStyleExpr
+
+
+# linters that we should / shouldn't run
+linters:
+  enable-all: true
+  disable:
+    - gochecknoglobals
+    - lll
+    - maligned
+    - prealloc
+    - wsl
+    - gomnd
+    - godox
+    - goerr113
+    - testpackage
+    - godot
+    - interfacer
+    - goheader
+    - noctx
+
+# rules to deal with reported isues
+issues:
+  # List of regexps of issue texts to exclude, empty list by default.
+  # But independently from this option we use default exclude patterns,
+  # it can be disabled by `exclude-use-default: false`. To list all
+  # excluded by default patterns execute `golangci-lint run --help`
+  exclude:
+
+  # Independently from option `exclude` we use default exclude patterns,
+  # it can be disabled by this option. To list all
+  # excluded by default patterns execute `golangci-lint run --help`.
+  # Default value for this option is true.
+  exclude-use-default: true
+
+  # Maximum issues count per one linter. Set to 0 to disable. Default is 50.
+  max-per-linter: 0
+
+  # Maximum count of issues with the same text. Set to 0 to disable. Default is 3.
+  max-same-issues: 0
+
+  # Show only new issues: if there are unstaged changes or untracked files,
+  # only those changes are analyzed, else only changes in HEAD~ are analyzed.
+  # It's a super-useful option for integration of golangci-lint into existing
+  # large codebase. It's not practical to fix all existing issues at the moment
+  # of integration: much better don't allow issues in new code.
+  # Default is false.
+  new: false
+
+  # Show only new issues created after git revision `REV`
+  new-from-rev: "HEAD~1"

.goreleaser.yml

@@ -0,0 +1,38 @@
+release:
+  prerelease: auto
+  name_template: "{{.ProjectName}} v{{.Version}}"
+
+builds:
+- env:
+    - CGO_ENABLED=0
+  goos:
+    - freebsd
+    - windows
+    - darwin
+    - linux
+    - dragonfly
+    - netbsd
+    - openbsd
+  goarch:
+    - amd64
+    - arm
+    - arm64
+    - 386
+  ldflags:
+    - -s -w -X main.githash={{.ShortCommit}}
+
+archives:
+  -
+    id: matterbridge
+    builds:
+    - matterbridge
+    name_template: "{{ .Binary }}-{{ .Version }}-{{ .Os }}-{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}"
+    format: binary
+    files:
+      - none*
+    replacements:
+      386: 32bit
+      amd64: 64bit
+
+checksum:
+  name_template: 'checksums.txt'

.travis.yml

@@ -1,51 +0,0 @@
-language: go
-go:
-    #- 1.7.x
-    - 1.10.x
-      # - tip
-
-# we have everything vendored
-install: true
-
-env:
-    - GOOS=linux GOARCH=amd64
-      #    - GOOS=windows GOARCH=amd64
-      #- GOOS=linux GOARCH=arm
-
-matrix:
-  # It's ok if our code fails on unstable development versions of Go.
-  allow_failures:
-    - go: tip
-  # Don't wait for tip tests to finish. Mark the test run green if the
-  # tests pass on the stable versions of Go.
-  fast_finish: true
-
-notifications:
-      email: false
-
-before_script:
-  - MY_VERSION=$(git describe --tags)
-  - GO_FILES=$(find . -iname '*.go' | grep -v /vendor/)  # All the .go files, excluding vendor/
-  - PKGS=$(go list ./... | grep -v /vendor/)             # All the import paths, excluding vendor/
-#  - go get github.com/golang/lint/golint                 # Linter
-  - go get honnef.co/go/tools/cmd/megacheck              # Badass static analyzer/linter
-
-# Anything in before_script: that returns a nonzero exit code will
-# flunk the build and immediately stop. It's sorta like having
-# set -e enabled in bash. 
-script:
- #- test -z $(gofmt -s -l $GO_FILES)  # Fail if a .go file hasn't been formatted with gofmt
-  - go test -v -race $PKGS            # Run all the tests with the race detector enabled
- #  - go vet $PKGS                      # go vet is the official Go static analyzer
-  - megacheck $PKGS                   # "go vet on steroids" + linter
-  - /bin/bash ci/bintray.sh
-  #- golint -set_exit_status $PKGS     # one last linter
-
-deploy:
-  provider: bintray
-  edge:
-    branch: v1.8.47
-  file: ci/deploy.json
-  user: 42wim
-  key:
-     secure: "CeXXe6JOmt7HYR81MdWLua0ltQHhDdkIeRGBFbgd7hkb1wi8eF9DgpAcQrTso8NIlHNZmSAP46uhFgsRvkuezzX0ygalZ7DCJyAyn3sAMEh+UQSHV1WGThRehTtidqRGjetzsIGSwdrJOWil+XTfbO1Z8DGzfakhSuAZka8CM4BAoe3YeP9rYK8h+84x0GHfczvsLtXZ3mWLvQuwe4pK6+ItBCUg0ae7O7ZUpWHy0xQQkkWztY/6RAzXfaG7DuGjIw+20fhx3WOXRNpHCtZ6Bc3qERCpk0s1HhlQWlrN9wDaFTBWYwlvSnNgvxxMbNXJ6RrRJ0l0bA7FUswYwyroxhzrGLdzWDg8dHaQkypocngdalfhpsnoO9j3ApJhomUFJ3UoEq5nOGRUrKn8MPi+dP0zE4kNQ3e4VNa1ufNrvfpWolMg3xh8OXuhQdD5wIM5zFAbRJLqWSCVAjPq4DDPecmvXBOlIial7oa312lN5qnBnUjvAcxszZ+FUyDHT1Grxzna4tMwxY9obPzZUzm7359AOCCwIQFVB8GLqD2nwIstcXS0zGRz+fhviPipHuBa02q5bGUZwmkvrSNab0s8Jo7pCrel2Rz3nWPKaiCfq2WjbW1CLheSMkOQrjsdUd1hhbqNWFPUjJPInTc77NAKCfm5runv5uyowRLh4NNd0sI="

Dockerfile

@@ -1,11 +1,16 @@
-FROM alpine:edge
+FROM alpine AS builder
-ENTRYPOINT ["/bin/matterbridge"]
 
 COPY . /go/src/github.com/42wim/matterbridge
-RUN apk update && apk add go git gcc musl-dev ca-certificates \
+RUN apk update && apk add go git gcc musl-dev \
         && cd /go/src/github.com/42wim/matterbridge \
         && export GOPATH=/go \
         && go get \
-        && go build -x -ldflags "-X main.githash=$(git log --pretty=format:'%h' -n 1)" -o /bin/matterbridge \
+        && go build -x -ldflags "-X main.githash=$(git log --pretty=format:'%h' -n 1)" -o /bin/matterbridge
-        && rm -rf /go \
+
-        && apk del --purge git go gcc musl-dev
+FROM alpine
+RUN apk --no-cache add ca-certificates mailcap
+COPY --from=builder /bin/matterbridge /bin/matterbridge
+RUN mkdir /etc/matterbridge \
+  && touch /etc/matterbridge/matterbridge.toml \
+  && ln -sf /matterbridge.toml /etc/matterbridge/matterbridge.toml
+ENTRYPOINT ["/bin/matterbridge", "-conf", "/etc/matterbridge/matterbridge.toml"]

README.md

@@ -1,104 +1,204 @@
+<div align="center">
+
 # matterbridge
-Click on one of the badges below to join the chat   
 
-[![Gitter](https://img.shields.io/gitter/room/nwjs/nw.js.svg?colorB=42f4242)](https://gitter.im/42wim/matterbridge) [![Join the IRC chat at https://webchat.freenode.net/?channels=matterbridgechat](https://img.shields.io/badge/IRC-matterbridgechat-green.svg?colorB=42f4242)](https://webchat.freenode.net/?channels=matterbridgechat) [![Discord](https://img.shields.io/badge/discord-matterbridge-green.svg?colorB=42f4242)](https://discord.gg/AkKPtrQ) [![Matrix](https://img.shields.io/badge/matrix-matterbridge-green.svg?colorB=42f4242)](https://riot.im/app/#/room/#matterbridge:matrix.org) [![Slack](https://img.shields.io/badge/slack-matterbridgechat-green.svg?colorB=42f4242)](https://join.slack.com/matterbridgechat/shared_invite/MjEwODMxNjU1NDMwLTE0OTk2MTU3NTMtMzZkZmRiNDZhOA) [![Mattermost](https://img.shields.io/badge/mattermost-matterbridge-green.svg?colorB=42f4242)](https://framateam.org/signup_user_complete/?id=tfqm33ggop8x3qgu4boeieta6e) [![Xmpp](https://img.shields.io/badge/xmpp-matterbridge@conference.jabber.de-green.svg?colorB=42f4242)](https://inverse.chat) [![Twitch](https://img.shields.io/badge/twitch-matterbridge-green.svg?colorB=42f4242)](https://www.twitch.tv/matterbridge) [![Zulip](https://img.shields.io/badge/zulip-matterbridge-green.svg?colorB=42f4242)](https://matterbridge.zulipchat.com/register/)
+![Matterbridge Logo](img/matterbridge-notext.gif)<br />
+**A simple chat bridge**<br />
+Letting people be where they want to be.<br />
+<sub>Bridges between a growing number of protocols. Click below to demo or join the development chat.</sub>
 
-[![Download stable](https://img.shields.io/github/release/42wim/matterbridge.svg?label=download%20stable)](https://github.com/42wim/matterbridge/releases/latest) [![Download dev](https://img.shields.io/bintray/v/42wim/nightly/Matterbridge.svg?label=download%20dev&colorB=007ec6)](https://bintray.com/42wim/nightly/Matterbridge/_latestVersion)
+   <sup>
 
-![matterbridge.gif](https://github.com/42wim/matterbridge/blob/master/img/matterbridge.gif)
+[Discord][mb-discord] |
+[Gitter][mb-gitter] |
+[IRC][mb-irc] |
+[Keybase][mb-keybase] |
+[Matrix][mb-matrix] |
+[Mattermost][mb-mattermost] |
+[MSTeams][mb-msteams] |
+[Rocket.Chat][mb-rocketchat] |
+[Slack][mb-slack] |
+[Telegram][mb-telegram] |
+[Twitch][mb-twitch] |
+[WhatsApp][mb-whatsapp] |
+[XMPP][mb-xmpp] |
+[Zulip][mb-zulip] |
+And more...
+</sup>
 
-Simple bridge between IRC, XMPP, Gitter, Mattermost, Slack, Discord, Telegram, Rocket.Chat, Hipchat(via xmpp), Matrix, Steam, ssh-chat and Zulip
+---
-Has a REST API.   
-Minecraft server chat support via [MatterLink](https://github.com/elytra/MatterLink)
 
-**Mattermost isn't required to run matterbridge. It bridges between any supported protocol.**   
+[![Download stable](https://img.shields.io/github/release/42wim/matterbridge.svg?label=download%20stable)](https://github.com/42wim/matterbridge/releases/latest)
-(The name matterbridge is a remnant when it was only bridging mattermost)
+[![Maintainability](https://api.codeclimate.com/v1/badges/82dff70ef2ba85a6173a/maintainability)](https://codeclimate.com/github/42wim/matterbridge/maintainability)
+[![Test Coverage](https://api.codeclimate.com/v1/badges/82dff70ef2ba85a6173a/test_coverage)](https://codeclimate.com/github/42wim/matterbridge/test_coverage)<br />
+
+  <hr />
+</div>
+<div align="right"><sup>
+
+**Note:** Matter<em>most</em> isn't required to run matter<em>bridge</em>.</sup></div>
+
+<p>
+  <a href="https://www.digitalocean.com/">
+    <img src="https://opensource.nyc3.cdn.digitaloceanspaces.com/attribution/assets/PoweredByDO/DO_Powered_by_Badge_blue.svg" width="201px">
+  </a>
+</p>
 
 # Table of Contents
- * [Features](https://github.com/42wim/matterbridge/wiki/Features)
- * [Requirements](#requirements)
- * [Screenshots](https://github.com/42wim/matterbridge/wiki/)
- * [Installing](#installing)
-   * [Binaries](#binaries)
-   * [Building](#building)
- * [Configuration](#configuration)
-   * [Howto](https://github.com/42wim/matterbridge/wiki/How-to-create-your-config)
-   * [Examples](#examples) 
- * [Running](#running)
-   * [Docker](#docker)
- * [Changelog](#changelog)
- * [FAQ](#faq)
- * [Thanks](#thanks)
 
-# Features
+- [matterbridge](#matterbridge)
-* [Support bridging between any protocols](https://github.com/42wim/matterbridge/wiki/Features#support-bridging-between-any-protocols)
+- [Table of Contents](#table-of-contents)
-* [Support multiple gateways(bridges) for your protocols](https://github.com/42wim/matterbridge/wiki/Features#support-multiple-gatewaysbridges-for-your-protocols)
+  - [Features](#features)
-* [Message edits and deletes](https://github.com/42wim/matterbridge/wiki/Features#message-edits-and-deletes)
+    - [Natively supported](#natively-supported)
-* [Attachment / files handling](https://github.com/42wim/matterbridge/wiki/Features#attachment--files-handling)
+    - [3rd party via matterbridge api](#3rd-party-via-matterbridge-api)
-* [Username and avatar spoofing](https://github.com/42wim/matterbridge/wiki/Features#username-and-avatar-spoofing)
+    - [API](#api)
-* [Private groups](https://github.com/42wim/matterbridge/wiki/Features#private-groups)
+  - [Chat with us](#chat-with-us)
-* [API](https://github.com/42wim/matterbridge/wiki/Features#api)
+  - [Screenshots](#screenshots)
+  - [Installing / upgrading](#installing--upgrading)
+    - [Binaries](#binaries)
+    - [Packages](#packages)
+  - [Building](#building)
+  - [Configuration](#configuration)
+    - [Basic configuration](#basic-configuration)
+    - [Settings](#settings)
+    - [Advanced configuration](#advanced-configuration)
+    - [Examples](#examples)
+      - [Bridge mattermost (off-topic) - irc (#testing)](#bridge-mattermost-off-topic---irc-testing)
+      - [Bridge slack (#general) - discord (general)](#bridge-slack-general---discord-general)
+  - [Running](#running)
+    - [Docker](#docker)
+  - [Changelog](#changelog)
+  - [FAQ](#faq)
+  - [Related projects](#related-projects)
+  - [Articles](#articles)
+  - [Thanks](#thanks)
 
-## API
+## Features
-The API is very basic at the moment and rather undocumented.
 
-Used by at least 2 projects. Feel free to make a PR to add your project to this list.
+- [Support bridging between any protocols](https://github.com/42wim/matterbridge/wiki/Features#support-bridging-between-any-protocols)
+- [Support multiple gateways(bridges) for your protocols](https://github.com/42wim/matterbridge/wiki/Features#support-multiple-gatewaysbridges-for-your-protocols)
+- [Message edits and deletes](https://github.com/42wim/matterbridge/wiki/Features#message-edits-and-deletes)
+- Preserves threading when possible
+- [Attachment / files handling](https://github.com/42wim/matterbridge/wiki/Features#attachment--files-handling)
+- [Username and avatar spoofing](https://github.com/42wim/matterbridge/wiki/Features#username-and-avatar-spoofing)
+- [Private groups](https://github.com/42wim/matterbridge/wiki/Features#private-groups)
+- [API](https://github.com/42wim/matterbridge/wiki/Features#api)
 
-* [MatterLink](https://github.com/elytra/MatterLink) (Matterbridge link for Minecraft Server chat)
+### Natively supported
-* [pyCord](https://github.com/NikkyAI/pyCord) (crossplatform chatbot)
 
-# Requirements
+- [Discord](https://discordapp.com)
-Accounts to one of the supported bridges
+- [Gitter](https://gitter.im)
-* [Mattermost](https://github.com/mattermost/platform/) 3.8.x - 3.10.x, 4.x, 5.x
+- [IRC](http://www.mirc.com/servers.html)
-* [IRC](http://www.mirc.com/servers.html)
+- [Keybase](https://keybase.io)
-* [XMPP](https://jabber.org)
+- [Matrix](https://matrix.org)
-* [Gitter](https://gitter.im)
+- [Mattermost](https://github.com/mattermost/mattermost-server/) 4.x, 5.x
-* [Slack](https://slack.com)
+- [Microsoft Teams](https://teams.microsoft.com)
-* [Discord](https://discordapp.com)
+- [Nextcloud Talk](https://nextcloud.com/talk/)
-* [Telegram](https://telegram.org)
+- [Rocket.chat](https://rocket.chat)
-* [Hipchat](https://www.hipchat.com)
+- [Slack](https://slack.com)
-* [Rocket.chat](https://rocket.chat)
+- [Ssh-chat](https://github.com/shazow/ssh-chat)
-* [Matrix](https://matrix.org)
+- [Steam](https://store.steampowered.com/)
-* [Steam](https://store.steampowered.com/)
+- [Telegram](https://telegram.org)
-* [Twitch](https://twitch.tv)
+- [Twitch](https://twitch.tv)
-* [Ssh-chat](https://github.com/shazow/ssh-chat)
+- [WhatsApp](https://www.whatsapp.com/)
-* [Zulip](https://zulipchat.com)
+- [XMPP](https://xmpp.org)
+- [Zulip](https://zulipchat.com)
 
-# Screenshots
+### 3rd party via matterbridge api
-See https://github.com/42wim/matterbridge/wiki
 
-# Installing
+- [Discourse](https://github.com/DeclanHoare/matterbabble)
-## Binaries
+- [Facebook messenger](https://github.com/VictorNine/fbridge)
-* Latest stable release [v1.11.0](https://github.com/42wim/matterbridge/releases/latest)
+- [Minecraft](https://github.com/elytra/MatterLink)
-* Development releases (follows master) can be downloaded [here](https://dl.bintray.com/42wim/nightly/)  
+- [Reddit](https://github.com/bonehurtingjuice/mattereddit)
+- [Counter-Strike, half-life and more](https://forums.alliedmods.net/showthread.php?t=319430)
+- [MatterAMXX](https://github.com/GabeIggy/MatterAMXX)
+
+### API
+
+The API is basic at the moment.
+More info and examples on the [wiki](https://github.com/42wim/matterbridge/wiki/Api).
+
+Used by the projects below. Feel free to make a PR to add your project to this list.
+
+- [MatterLink](https://github.com/elytra/MatterLink) (Matterbridge link for Minecraft Server chat)
+- [pyCord](https://github.com/NikkyAI/pyCord) (crossplatform chatbot)
+- [Mattereddit](https://github.com/bonehurtingjuice/mattereddit) (Reddit chat support)
+- [fbridge](https://github.com/VictorNine/fbridge) (Facebook messenger support)
+- [matterbabble](https://github.com/DeclanHoare/matterbabble) (Discourse support)
+- [MatterAMXX](https://forums.alliedmods.net/showthread.php?t=319430) (Counter-Strike, half-life and more via AMXX mod)
+
+## Chat with us
+
+Questions or want to test on your favorite platform? Join below:
+
+- [Discord][mb-discord]
+- [Gitter][mb-gitter]
+- [IRC][mb-irc]
+- [Keybase][mb-keybase]
+- [Matrix][mb-matrix]
+- [Mattermost][mb-mattermost]
+- [Rocket.Chat][mb-rocketchat]
+- [Slack][mb-slack]
+- [Telegram][mb-telegram]
+- [Twitch][mb-twitch]
+- [XMPP][mb-xmpp] (matterbridge@conference.jabber.de)
+- [Zulip][mb-zulip]
+
+## Screenshots
+
+See <https://github.com/42wim/matterbridge/wiki>
+
+## Installing / upgrading
+
+### Binaries
+
+- Latest stable release [v1.18.3](https://github.com/42wim/matterbridge/releases/latest)
+- Development releases (follows master) can be downloaded [here](https://github.com/42wim/matterbridge/actions) selecting the latest green build and then artifacts.
+
+To install or upgrade just download the latest [binary](https://github.com/42wim/matterbridge/releases/latest) and follow the instructions on the [howto](https://github.com/42wim/matterbridge/wiki/How-to-create-your-config) for a step by step walkthrough for creating your configuration.
+
+### Packages
+
+- [Overview](https://repology.org/metapackage/matterbridge/versions)
+- [snap](https://snapcraft.io/matterbridge)
 
 ## Building
-Go 1.8+ is required. Make sure you have [Go](https://golang.org/doc/install) properly installed, including setting up your [GOPATH](https://golang.org/doc/code.html#GOPATH).
 
-After Go is setup, download matterbridge to your $GOPATH directory. 
+Most people just want to use binaries, you can find those [here](https://github.com/42wim/matterbridge/releases/latest)
 
-```
+If you really want to build from source, follow these instructions:
-cd $GOPATH
+Go 1.12+ is required. Make sure you have [Go](https://golang.org/doc/install) properly installed.
+
+```bash
 go get github.com/42wim/matterbridge
 ```
 
-You should now have matterbridge binary in the bin directory:
+You should now have matterbridge binary in the ~/go/bin directory:
 
-```
+```bash
-$ ls bin/
+$ ls ~/go/bin/
 matterbridge
 ```
 
-# Configuration
+## Configuration
-## Basic configuration
+
+### Basic configuration
+
 See [howto](https://github.com/42wim/matterbridge/wiki/How-to-create-your-config) for a step by step walkthrough for creating your configuration.
 
-## Advanced configuration
+### Settings
-* [matterbridge.toml.sample](https://github.com/42wim/matterbridge/blob/master/matterbridge.toml.sample) for documentation and an example.
 
-## Examples 
+All possible [settings](https://github.com/42wim/matterbridge/wiki/Settings) for each bridge.
-### Bridge mattermost (off-topic) - irc (#testing)
+
-```
+### Advanced configuration
+
+- [matterbridge.toml.sample](https://github.com/42wim/matterbridge/blob/master/matterbridge.toml.sample) for documentation and an example.
+
+### Examples
+
+#### Bridge mattermost (off-topic) - irc (#testing)
+
+```toml
 [irc]
     [irc.freenode]
     Server="irc.freenode.net:6667"
@@ -125,8 +225,9 @@ enable=true
     channel="off-topic"
 ```
 
-### Bridge slack (#general) - discord (general)
+#### Bridge slack (#general) - discord (general)
-```
+
+```toml
 [slack]
 [slack.test]
 Token="yourslacktoken"
@@ -153,11 +254,11 @@ RemoteNickFormat="[{PROTOCOL}/{BRIDGE}] <{NICK}> "
     channel = "general"
 ```
 
-# Running
+## Running
 
 See [howto](https://github.com/42wim/matterbridge/wiki/How-to-create-your-config) for a step by step walkthrough for creating your configuration.
 
-```
+```bash
 Usage of ./matterbridge:
   -conf string
         config file (default "matterbridge.toml")
@@ -169,37 +270,88 @@ Usage of ./matterbridge:
         show version
 ```
 
-## Docker
+### Docker
-Create your matterbridge.toml file locally eg in ```/tmp/matterbridge.toml```
+
-```
+Please take a look at the [Docker Wiki page](https://github.com/42wim/matterbridge/wiki/Deploy:-Docker) for more information.
-docker run -ti -v /tmp/matterbridge.toml:/matterbridge.toml 42wim/matterbridge
+
-```
+## Changelog
 
-# Changelog
 See [changelog.md](https://github.com/42wim/matterbridge/blob/master/changelog.md)
 
-# FAQ
+## FAQ
 
 See [FAQ](https://github.com/42wim/matterbridge/wiki/FAQ)
 
-Want to tip ? 
+## Related projects
-* eth: 0xb3f9b5387c66ad6be892bcb7bbc67862f3abc16f
-* btc: 1N7cKHj5SfqBHBzDJ6kad4BzeqUBBS2zhs
 
-# Thanks
+- [jwflory/ansible-role-matterbridge](https://galaxy.ansible.com/jwflory/matterbridge) (Ansible role to simplify deploying Matterbridge)
-[![Digitalocean](https://snag.gy/3LVifX.jpg)](https://www.digitalocean.com/) for sponsoring demo/testing droplets.
+- [matterbridge autoconfig](https://github.com/patcon/matterbridge-autoconfig)
+- [matterbridge config viewer](https://github.com/patcon/matterbridge-heroku-viewer)
+- [matterbridge-heroku](https://github.com/cadecairos/matterbridge-heroku)
+- [mattereddit](https://github.com/bonehurtingjuice/mattereddit)
+- [matterlink](https://github.com/elytra/MatterLink)
+- [mattermost-plugin](https://github.com/matterbridge/mattermost-plugin) - Run matterbridge as a plugin in mattermost
+- [pyCord](https://github.com/NikkyAI/pyCord) (crossplatform chatbot)
+- [fbridge](https://github.com/VictorNine/fbridge) (Facebook messenger support)
+- [isla](https://github.com/alphachung/isla) (Bot for Discord-Telegram groups used alongside matterbridge)
+- [matterbabble](https://github.com/DeclanHoare/matterbabble) (Connect Discourse threads to Matterbridge)
+
+## Articles
+
+- [matterbridge on kubernetes](https://medium.freecodecamp.org/using-kubernetes-to-deploy-a-chat-gateway-or-when-technology-works-like-its-supposed-to-a169a8cd69a3)
+- <https://mattermost.com/blog/connect-irc-to-mattermost/>
+- <https://blog.valvin.fr/2016/09/17/mattermost-et-un-channel-irc-cest-possible/>
+- <https://blog.brightscout.com/top-10-mattermost-integrations/>
+- <https://www.algoo.fr/blog/2018/01/19/recouvrez-votre-liberte-en-quittant-slack-pour-un-mattermost-auto-heberge/>
+- <https://kopano.com/blog/matterbridge-bridging-mattermost-chat/>
+- <https://www.stitcher.com/s/?eid=52382713>
+- <https://daniele.tech/2019/02/how-to-use-matterbridge-to-connect-2-different-slack-workspaces/>
+- <https://userlinux.net/mattermost-and-matterbridge.html>
+
+## Thanks
+
+<p>This project is supported by:</p>
+<p>
+  <a href="https://www.digitalocean.com/">
+    <img src="https://opensource.nyc3.cdn.digitaloceanspaces.com/attribution/assets/SVG/DO_Logo_horizontal_blue.svg" width="201px">
+  </a>
+</p>
 
 Matterbridge wouldn't exist without these libraries:
-* discord - https://github.com/bwmarrin/discordgo
+
-* echo - https://github.com/labstack/echo
+- discord - <https://github.com/bwmarrin/discordgo>
-* gitter - https://github.com/sromku/go-gitter
+- echo - <https://github.com/labstack/echo>
-* gops - https://github.com/google/gops
+- gitter - <https://github.com/sromku/go-gitter>
-* gozulipbot - https://github.com/ifo/gozulipbot
+- gops - <https://github.com/google/gops>
-* irc - https://github.com/lrstanley/girc
+- gozulipbot - <https://github.com/ifo/gozulipbot>
-* mattermost - https://github.com/mattermost/platform
+- irc - <https://github.com/lrstanley/girc>
-* matrix - https://github.com/matrix-org/gomatrix
+- keybase - <https://github.com/keybase/go-keybase-chat-bot>
-* slack - https://github.com/nlopes/slack
+- matrix - <https://github.com/matrix-org/gomatrix>
-* steam - https://github.com/Philipp15b/go-steam
+- mattermost - <https://github.com/mattermost/mattermost-server>
-* telegram - https://github.com/go-telegram-bot-api/telegram-bot-api
+- msgraph.go - <https://github.com/yaegashi/msgraph.go>
-* xmpp - https://github.com/mattn/go-xmpp
+- nctalk - <https://github.com/gary-kim/go-nc-talk>
-* zulip - https://github.com/ifo/gozulipbot
+- slack - <https://github.com/nlopes/slack>
+- sshchat - <https://github.com/shazow/ssh-chat>
+- steam - <https://github.com/Philipp15b/go-steam>
+- telegram - <https://github.com/go-telegram-bot-api/telegram-bot-api>
+- tengo - <https://github.com/d5/tengo>
+- whatsapp - <https://github.com/Rhymen/go-whatsapp>
+- xmpp - <https://github.com/mattn/go-xmpp>
+- zulip - <https://github.com/ifo/gozulipbot>
+
+<!-- Links -->
+
+[mb-discord]: https://discord.gg/AkKPtrQ
+[mb-gitter]: https://gitter.im/42wim/matterbridge
+[mb-irc]: https://webchat.freenode.net/?channels=matterbridgechat
+[mb-keybase]: https://keybase.io/team/matterbridge
+[mb-matrix]: https://riot.im/app/#/room/#matterbridge:matrix.org
+[mb-mattermost]: https://framateam.org/signup_user_complete/?id=tfqm33ggop8x3qgu4boeieta6e
+[mb-msteams]: https://teams.microsoft.com/join/hj92x75gd3y7
+[mb-rocketchat]: https://open.rocket.chat/channel/matterbridge
+[mb-slack]: https://join.slack.com/matterbridgechat/shared_invite/MjEwODMxNjU1NDMwLTE0OTk2MTU3NTMtMzZkZmRiNDZhOA
+[mb-telegram]: https://t.me/Matterbridge
+[mb-twitch]: https://www.twitch.tv/matterbridge
+[mb-whatsapp]: https://www.whatsapp.com/
+[mb-xmpp]: https://inverse.chat/
+[mb-zulip]: https://matterbridge.zulipchat.com/register/

bridge/api/api.go

@@ -6,20 +6,23 @@ import (
 	"sync"
 	"time"
 
+	"gopkg.in/olahol/melody.v1"
+
 	"github.com/42wim/matterbridge/bridge"
 	"github.com/42wim/matterbridge/bridge/config"
-	"github.com/labstack/echo"
+	"github.com/labstack/echo/v4"
-	"github.com/labstack/echo/middleware"
+	"github.com/labstack/echo/v4/middleware"
-	"github.com/zfjagann/golang-ring"
+	ring "github.com/zfjagann/golang-ring"
 )
 
-type Api struct {
+type API struct {
 	Messages ring.Ring
 	sync.RWMutex
 	*bridge.Config
+	mrouter *melody.Melody
 }
 
-type ApiMessage struct {
+type Message struct {
 	Text     string `json:"text"`
 	Username string `json:"username"`
 	UserID   string `json:"userid"`
@@ -28,19 +31,56 @@ type ApiMessage struct {
 }
 
 func New(cfg *bridge.Config) bridge.Bridger {
-	b := &Api{Config: cfg}
+	b := &API{Config: cfg}
 	e := echo.New()
 	e.HideBanner = true
 	e.HidePort = true
+
+	b.mrouter = melody.New()
+	b.mrouter.HandleMessage(func(s *melody.Session, msg []byte) {
+		message := config.Message{}
+		err := json.Unmarshal(msg, &message)
+		if err != nil {
+			b.Log.Errorf("failed to decode message from byte[] '%s'", string(msg))
+			return
+		}
+		b.handleWebsocketMessage(message, s)
+	})
+	b.mrouter.HandleConnect(func(session *melody.Session) {
+		greet := b.getGreeting()
+		data, err := json.Marshal(greet)
+		if err != nil {
+			b.Log.Errorf("failed to encode message '%v'", greet)
+			return
+		}
+		err = session.Write(data)
+		if err != nil {
+			b.Log.Errorf("failed to write message '%s'", string(data))
+			return
+		}
+		// TODO: send message history buffer from `b.Messages` here
+	})
+
 	b.Messages = ring.Ring{}
-	b.Messages.SetCapacity(b.GetInt("Buffer"))
+	if b.GetInt("Buffer") != 0 {
+		b.Messages.SetCapacity(b.GetInt("Buffer"))
+	}
 	if b.GetString("Token") != "" {
 		e.Use(middleware.KeyAuth(func(key string, c echo.Context) (bool, error) {
 			return key == b.GetString("Token"), nil
 		}))
 	}
+
+	// Set RemoteNickFormat to a sane default
+	if !b.IsKeySet("RemoteNickFormat") {
+		b.Log.Debugln("RemoteNickFormat is unset, defaulting to \"{NICK}\"")
+		b.Config.Config.Viper().Set(b.GetConfigKey("RemoteNickFormat"), "{NICK}")
+	}
+
+	e.GET("/api/health", b.handleHealthcheck)
 	e.GET("/api/messages", b.handleMessages)
 	e.GET("/api/stream", b.handleStream)
+	e.GET("/api/websocket", b.handleWebsocket)
 	e.POST("/api/message", b.handlePostMessage)
 	go func() {
 		if b.GetString("BindAddress") == "" {
@@ -52,30 +92,41 @@ func New(cfg *bridge.Config) bridge.Bridger {
 	return b
 }
 
-func (b *Api) Connect() error {
+func (b *API) Connect() error {
 	return nil
 }
-func (b *Api) Disconnect() error {
-	return nil
 
-}
+func (b *API) Disconnect() error {
-func (b *Api) JoinChannel(channel config.ChannelInfo) error {
 	return nil
-
 }
 
-func (b *Api) Send(msg config.Message) (string, error) {
+func (b *API) JoinChannel(channel config.ChannelInfo) error {
+	return nil
+}
+
+func (b *API) Send(msg config.Message) (string, error) {
 	b.Lock()
 	defer b.Unlock()
 	// ignore delete messages
-	if msg.Event == config.EVENT_MSG_DELETE {
+	if msg.Event == config.EventMsgDelete {
 		return "", nil
 	}
-	b.Messages.Enqueue(&msg)
+	b.Log.Debugf("enqueueing message from %s on ring buffer", msg.Username)
+	b.Messages.Enqueue(msg)
+
+	data, err := json.Marshal(msg)
+	if err != nil {
+		b.Log.Errorf("failed to encode message  '%s'", msg)
+	}
+	_ = b.mrouter.Broadcast(data)
 	return "", nil
 }
 
-func (b *Api) handlePostMessage(c echo.Context) error {
+func (b *API) handleHealthcheck(c echo.Context) error {
+	return c.String(http.StatusOK, "OK")
+}
+
+func (b *API) handlePostMessage(c echo.Context) error {
 	message := config.Message{}
 	if err := c.Bind(&message); err != nil {
 		return err
@@ -91,7 +142,7 @@ func (b *Api) handlePostMessage(c echo.Context) error {
 	return c.JSON(http.StatusOK, message)
 }
 
-func (b *Api) handleMessages(c echo.Context) error {
+func (b *API) handleMessages(c echo.Context) error {
 	b.Lock()
 	defer b.Unlock()
 	c.JSONPretty(http.StatusOK, b.Messages.Values(), " ")
@@ -99,23 +150,58 @@ func (b *Api) handleMessages(c echo.Context) error {
 	return nil
 }
 
-func (b *Api) handleStream(c echo.Context) error {
+func (b *API) getGreeting() config.Message {
-	c.Response().Header().Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
+	return config.Message{
-	c.Response().WriteHeader(http.StatusOK)
+		Event:     config.EventAPIConnected,
-	closeNotifier := c.Response().CloseNotify()
+		Timestamp: time.Now(),
-	for {
-		select {
-		case <-closeNotifier:
-			return nil
-		default:
-			msg := b.Messages.Dequeue()
-			if msg != nil {
-				if err := json.NewEncoder(c.Response()).Encode(msg); err != nil {
-					return err
-				}
-				c.Response().Flush()
-			}
-			time.Sleep(200 * time.Millisecond)
-		}
 	}
 }
+
+func (b *API) handleStream(c echo.Context) error {
+	c.Response().Header().Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
+	c.Response().WriteHeader(http.StatusOK)
+	greet := b.getGreeting()
+	if err := json.NewEncoder(c.Response()).Encode(greet); err != nil {
+		return err
+	}
+	c.Response().Flush()
+	for {
+		// TODO: this causes issues, messages should be broadcasted to all connected clients
+		msg := b.Messages.Dequeue()
+		if msg != nil {
+			if err := json.NewEncoder(c.Response()).Encode(msg); err != nil {
+				return err
+			}
+			c.Response().Flush()
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+}
+
+func (b *API) handleWebsocketMessage(message config.Message, s *melody.Session) {
+	message.Channel = "api"
+	message.Protocol = "api"
+	message.Account = b.Account
+	message.ID = ""
+	message.Timestamp = time.Now()
+
+	data, err := json.Marshal(message)
+	if err != nil {
+		b.Log.Errorf("failed to encode message for loopback '%v'", message)
+		return
+	}
+	_ = b.mrouter.BroadcastOthers(data, s)
+
+	b.Log.Debugf("Sending websocket message from %s on %s to gateway", message.Username, "api")
+	b.Remote <- message
+}
+
+func (b *API) handleWebsocket(c echo.Context) error {
+	err := b.mrouter.HandleRequest(c.Response(), c.Request())
+	if err != nil {
+		b.Log.Errorf("error in websocket handling  '%v'", err)
+		return err
+	}
+
+	return nil
+}

bridge/bridge.go

@@ -1,10 +1,13 @@
 package bridge
 
 import (
-	"github.com/42wim/matterbridge/bridge/config"
+	"log"
-	log "github.com/sirupsen/logrus"
-
 	"strings"
+	"sync"
+	"time"
+
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/sirupsen/logrus"
 )
 
 type Bridger interface {
@@ -16,48 +19,63 @@ type Bridger interface {
 
 type Bridge struct {
 	Bridger
-	Name     string
+	*sync.RWMutex
-	Account  string
+
-	Protocol string
+	Name           string
-	Channels map[string]config.ChannelInfo
+	Account        string
-	Joined   map[string]bool
+	Protocol       string
-	Log      *log.Entry
+	Channels       map[string]config.ChannelInfo
-	Config   *config.Config
+	Joined         map[string]bool
-	General  *config.Protocol
+	ChannelMembers *config.ChannelMembers
+	Log            *logrus.Entry
+	Config         config.Config
+	General        *config.Protocol
 }
 
 type Config struct {
-	//	General *config.Protocol
-	Remote chan config.Message
-	Log    *log.Entry
 	*Bridge
+
+	Remote chan config.Message
 }
 
 // Factory is the factory function to create a bridge
 type Factory func(*Config) Bridger
 
 func New(bridge *config.Bridge) *Bridge {
-	b := new(Bridge)
-	b.Channels = make(map[string]config.ChannelInfo)
 	accInfo := strings.Split(bridge.Account, ".")
+	if len(accInfo) != 2 {
+		log.Fatalf("config failure, account incorrect: %s", bridge.Account)
+	}
+
 	protocol := accInfo[0]
 	name := accInfo[1]
-	b.Name = name
+
-	b.Protocol = protocol
+	return &Bridge{
-	b.Account = bridge.Account
+		RWMutex:  new(sync.RWMutex),
-	b.Joined = make(map[string]bool)
+		Channels: make(map[string]config.ChannelInfo),
-	return b
+		Name:     name,
+		Protocol: protocol,
+		Account:  bridge.Account,
+		Joined:   make(map[string]bool),
+	}
 }
 
 func (b *Bridge) JoinChannels() error {
-	err := b.joinChannels(b.Channels, b.Joined)
+	return b.joinChannels(b.Channels, b.Joined)
-	return err
+}
+
+// SetChannelMembers sets the newMembers to the bridge ChannelMembers
+func (b *Bridge) SetChannelMembers(newMembers *config.ChannelMembers) {
+	b.Lock()
+	b.ChannelMembers = newMembers
+	b.Unlock()
 }
 
 func (b *Bridge) joinChannels(channels map[string]config.ChannelInfo, exists map[string]bool) error {
 	for ID, channel := range channels {
 		if !exists[ID] {
 			b.Log.Infof("%s: joining %s (ID: %s)", b.Account, channel.Name, ID)
+			time.Sleep(time.Duration(b.GetInt("JoinDelay")) * time.Millisecond)
 			err := b.JoinChannel(channel)
 			if err != nil {
 				return err
@@ -68,37 +86,50 @@ func (b *Bridge) joinChannels(channels map[string]config.ChannelInfo, exists map
 	return nil
 }
 
+func (b *Bridge) GetConfigKey(key string) string {
+	return b.Account + "." + key
+}
+
+func (b *Bridge) IsKeySet(key string) bool {
+	return b.Config.IsKeySet(b.GetConfigKey(key)) || b.Config.IsKeySet("general."+key)
+}
+
 func (b *Bridge) GetBool(key string) bool {
-	if b.Config.GetBool(b.Account + "." + key) {
+	val, ok := b.Config.GetBool(b.GetConfigKey(key))
-		return b.Config.GetBool(b.Account + "." + key)
+	if !ok {
+		val, _ = b.Config.GetBool("general." + key)
 	}
-	return b.Config.GetBool("general." + key)
+	return val
 }
 
 func (b *Bridge) GetInt(key string) int {
-	if b.Config.GetInt(b.Account+"."+key) != 0 {
+	val, ok := b.Config.GetInt(b.GetConfigKey(key))
-		return b.Config.GetInt(b.Account + "." + key)
+	if !ok {
+		val, _ = b.Config.GetInt("general." + key)
 	}
-	return b.Config.GetInt("general." + key)
+	return val
 }
 
 func (b *Bridge) GetString(key string) string {
-	if b.Config.GetString(b.Account+"."+key) != "" {
+	val, ok := b.Config.GetString(b.GetConfigKey(key))
-		return b.Config.GetString(b.Account + "." + key)
+	if !ok {
+		val, _ = b.Config.GetString("general." + key)
 	}
-	return b.Config.GetString("general." + key)
+	return val
 }
 
 func (b *Bridge) GetStringSlice(key string) []string {
-	if len(b.Config.GetStringSlice(b.Account+"."+key)) != 0 {
+	val, ok := b.Config.GetStringSlice(b.GetConfigKey(key))
-		return b.Config.GetStringSlice(b.Account + "." + key)
+	if !ok {
+		val, _ = b.Config.GetStringSlice("general." + key)
 	}
-	return b.Config.GetStringSlice("general." + key)
+	return val
 }
 
 func (b *Bridge) GetStringSlice2D(key string) [][]string {
-	if len(b.Config.GetStringSlice2D(b.Account+"."+key)) != 0 {
+	val, ok := b.Config.GetStringSlice2D(b.GetConfigKey(key))
-		return b.Config.GetStringSlice2D(b.Account + "." + key)
+	if !ok {
+		val, _ = b.Config.GetStringSlice2D("general." + key)
 	}
-	return b.Config.GetStringSlice2D("general." + key)
+	return val
 }

bridge/config/config.go

@@ -2,26 +2,30 @@ package config
 
 import (
 	"bytes"
+	"io/ioutil"
 	"os"
+	"path/filepath"
 	"strings"
 	"sync"
 	"time"
 
 	"github.com/fsnotify/fsnotify"
-	log "github.com/sirupsen/logrus"
+	"github.com/sirupsen/logrus"
 	"github.com/spf13/viper"
-	prefixed "github.com/x-cray/logrus-prefixed-formatter"
 )
 
 const (
-	EVENT_JOIN_LEAVE        = "join_leave"
+	EventJoinLeave         = "join_leave"
-	EVENT_TOPIC_CHANGE      = "topic_change"
+	EventTopicChange       = "topic_change"
-	EVENT_FAILURE           = "failure"
+	EventFailure           = "failure"
-	EVENT_FILE_FAILURE_SIZE = "file_failure_size"
+	EventFileFailureSize   = "file_failure_size"
-	EVENT_AVATAR_DOWNLOAD   = "avatar_download"
+	EventAvatarDownload    = "avatar_download"
-	EVENT_REJOIN_CHANNELS   = "rejoin_channels"
+	EventRejoinChannels    = "rejoin_channels"
-	EVENT_USER_ACTION       = "user_action"
+	EventUserAction        = "user_action"
-	EVENT_MSG_DELETE        = "msg_delete"
+	EventMsgDelete         = "msg_delete"
+	EventAPIConnected      = "api_connected"
+	EventUserTyping        = "user_typing"
+	EventGetChannelMembers = "get_channel_members"
 )
 
 type Message struct {
@@ -34,6 +38,7 @@ type Message struct {
 	Event     string    `json:"event"`
 	Protocol  string    `json:"protocol"`
 	Gateway   string    `json:"gateway"`
+	ParentID  string    `json:"parent_id"`
 	Timestamp time.Time `json:"timestamp"`
 	ID        string    `json:"id"`
 	Extra     map[string][]interface{}
@@ -58,27 +63,45 @@ type ChannelInfo struct {
 	Options     ChannelOptions
 }
 
+type ChannelMember struct {
+	Username    string
+	Nick        string
+	UserID      string
+	ChannelID   string
+	ChannelName string
+}
+
+type ChannelMembers []ChannelMember
+
 type Protocol struct {
 	AuthCode               string // steam
 	BindAddress            string // mattermost, slack // DEPRECATED
 	Buffer                 int    // api
 	Charset                string // irc
+	ClientID               string // msteams
 	ColorNicks             bool   // only irc for now
 	Debug                  bool   // general
 	DebugLevel             int    // only for irc now
+	DisableWebPagePreview  bool   // telegram
 	EditSuffix             string // mattermost, slack, discord, telegram, gitter
 	EditDisable            bool   // mattermost, slack, discord, telegram, gitter
+	HTMLDisable            bool   // matrix
 	IconURL                string // mattermost, slack
+	IgnoreFailureOnStart   bool   // general
 	IgnoreNicks            string // all protocols
 	IgnoreMessages         string // all protocols
 	Jid                    string // xmpp
+	JoinDelay              string // all protocols
 	Label                  string // all protocols
 	Login                  string // mattermost, matrix
+	LogFile                string // general
 	MediaDownloadBlackList []string
 	MediaDownloadPath      string // Basically MediaServerUpload, but instead of uploading it, just write it to a file on the same server.
 	MediaDownloadSize      int    // all protocols
 	MediaServerDownload    string
 	MediaServerUpload      string
+	MediaConvertTgs        string     // telegram
+	MediaConvertWebPToPNG  bool       // telegram
 	MessageDelay           int        // IRC, time in millisecond to wait between messages
 	MessageFormat          string     // telegram
 	MessageLength          int        // IRC, max length of a message allowed
@@ -97,37 +120,51 @@ type Protocol struct {
 	NoTLS                  bool       // mattermost
 	Password               string     // IRC,mattermost,XMPP,matrix
 	PrefixMessagesWithNick bool       // mattemost, slack
+	PreserveThreading      bool       // slack
 	Protocol               string     // all protocols
 	QuoteDisable           bool       // telegram
 	QuoteFormat            string     // telegram
+	QuoteLengthLimit       int        // telegram
 	RejoinDelay            int        // IRC
 	ReplaceMessages        [][]string // all protocols
 	ReplaceNicks           [][]string // all protocols
 	RemoteNickFormat       string     // all protocols
+	RunCommands            []string   // IRC
 	Server                 string     // IRC,mattermost,XMPP,discord
+	SessionFile            string     // msteams,whatsapp
 	ShowJoinPart           bool       // all protocols
 	ShowTopicChange        bool       // slack
+	ShowUserTyping         bool       // slack
 	ShowEmbeds             bool       // discord
 	SkipTLSVerify          bool       // IRC, mattermost
+	SkipVersionCheck       bool       // mattermost
 	StripNick              bool       // all protocols
-	Team                   string     // mattermost
+	StripMarkdown          bool       // irc
+	SyncTopic              bool       // slack
+	TengoModifyMessage     string     // general
+	Team                   string     // mattermost, keybase
+	TeamID                 string     // msteams
+	TenantID               string     // msteams
 	Token                  string     // gitter, slack, discord, api
 	Topic                  string     // zulip
 	URL                    string     // mattermost, slack // DEPRECATED
 	UseAPI                 bool       // mattermost, slack
+	UseLocalAvatar         []string   // discord
 	UseSASL                bool       // IRC
 	UseTLS                 bool       // IRC
+	UseDiscriminator       bool       // discord
 	UseFirstName           bool       // telegram
 	UseUserName            bool       // discord
 	UseInsecureURL         bool       // telegram
+	VerboseJoinPart        bool       // IRC
 	WebhookBindAddress     string     // mattermost, slack
 	WebhookURL             string     // mattermost, slack
-	WebhookUse             string     // mattermost, slack, discord
 }
 
 type ChannelOptions struct {
 	Key        string // irc, xmpp
 	WebhookURL string // discord
+	Topic      string // zulip
 }
 
 type Bridge struct {
@@ -145,6 +182,13 @@ type Gateway struct {
 	InOut  []Bridge
 }
 
+type Tengo struct {
+	InMessage        string
+	Message          string
+	RemoteNickFormat string
+	OutMessage       string
+}
+
 type SameChannelGateway struct {
 	Name     string
 	Enable   bool
@@ -152,127 +196,173 @@ type SameChannelGateway struct {
 	Accounts []string
 }
 
-type ConfigValues struct {
+type BridgeValues struct {
-	Api                map[string]Protocol
+	API                map[string]Protocol
-	Irc                map[string]Protocol
+	IRC                map[string]Protocol
 	Mattermost         map[string]Protocol
 	Matrix             map[string]Protocol
 	Slack              map[string]Protocol
+	SlackLegacy        map[string]Protocol
 	Steam              map[string]Protocol
 	Gitter             map[string]Protocol
-	Xmpp               map[string]Protocol
+	XMPP               map[string]Protocol
 	Discord            map[string]Protocol
 	Telegram           map[string]Protocol
 	Rocketchat         map[string]Protocol
-	Sshchat            map[string]Protocol
+	SSHChat            map[string]Protocol
+	WhatsApp           map[string]Protocol // TODO is this struct used? Search for "SlackLegacy" for example didn't return any results
 	Zulip              map[string]Protocol
+	Keybase            map[string]Protocol
 	General            Protocol
+	Tengo              Tengo
 	Gateway            []Gateway
 	SameChannelGateway []SameChannelGateway
 }
 
-type Config struct {
+type Config interface {
-	v *viper.Viper
+	Viper() *viper.Viper
-	*ConfigValues
+	BridgeValues() *BridgeValues
-	sync.RWMutex
+	IsKeySet(key string) bool
+	GetBool(key string) (bool, bool)
+	GetInt(key string) (int, bool)
+	GetString(key string) (string, bool)
+	GetStringSlice(key string) ([]string, bool)
+	GetStringSlice2D(key string) ([][]string, bool)
 }
 
-func NewConfig(cfgfile string) *Config {
+type config struct {
-	log.SetFormatter(&prefixed.TextFormatter{PrefixPadding: 13, DisableColors: true, FullTimestamp: false})
+	sync.RWMutex
-	flog := log.WithFields(log.Fields{"prefix": "config"})
+
-	var cfg ConfigValues
+	logger *logrus.Entry
-	viper.SetConfigType("toml")
+	v      *viper.Viper
+	cv     *BridgeValues
+}
+
+// NewConfig instantiates a new configuration based on the specified configuration file path.
+func NewConfig(rootLogger *logrus.Logger, cfgfile string) Config {
+	logger := rootLogger.WithFields(logrus.Fields{"prefix": "config"})
+
 	viper.SetConfigFile(cfgfile)
-	viper.SetEnvPrefix("matterbridge")
+	input, err := ioutil.ReadFile(cfgfile)
-	viper.AddConfigPath(".")
-	viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
-	viper.AutomaticEnv()
-	f, err := os.Open(cfgfile)
 	if err != nil {
-		log.Fatal(err)
+		logger.Fatalf("Failed to read configuration file: %#v", err)
 	}
-	err = viper.ReadConfig(f)
+
-	if err != nil {
+	cfgtype := detectConfigType(cfgfile)
-		log.Fatal(err)
+	mycfg := newConfigFromString(logger, input, cfgtype)
+	if mycfg.cv.General.LogFile != "" {
+		logfile, err := os.OpenFile(mycfg.cv.General.LogFile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600)
+		if err == nil {
+			logger.Info("Opening log file ", mycfg.cv.General.LogFile)
+			rootLogger.Out = logfile
+		} else {
+			logger.Warn("Failed to open ", mycfg.cv.General.LogFile)
+		}
 	}
-	err = viper.Unmarshal(&cfg)
+	if mycfg.cv.General.MediaDownloadSize == 0 {
-	if err != nil {
+		mycfg.cv.General.MediaDownloadSize = 1000000
-		log.Fatal("blah", err)
-	}
-	mycfg := new(Config)
-	mycfg.v = viper.GetViper()
-	if cfg.General.MediaDownloadSize == 0 {
-		cfg.General.MediaDownloadSize = 1000000
 	}
 	viper.WatchConfig()
 	viper.OnConfigChange(func(e fsnotify.Event) {
-		flog.Println("Config file changed:", e.Name)
+		logger.Println("Config file changed:", e.Name)
 	})
-
-	mycfg.ConfigValues = &cfg
 	return mycfg
 }
 
-func NewConfigFromString(input []byte) *Config {
+// detectConfigType detects JSON and YAML formats, defaults to TOML.
-	var cfg ConfigValues
+func detectConfigType(cfgfile string) string {
-	viper.SetConfigType("toml")
+	fileExt := filepath.Ext(cfgfile)
-	err := viper.ReadConfig(bytes.NewBuffer(input))
+	switch fileExt {
-	if err != nil {
+	case ".json":
-		log.Fatal(err)
+		return "json"
+	case ".yaml", ".yml":
+		return "yaml"
 	}
-	err = viper.Unmarshal(&cfg)
+	return "toml"
-	if err != nil {
+}
-		log.Fatal(err)
+
+// NewConfigFromString instantiates a new configuration based on the specified string.
+func NewConfigFromString(rootLogger *logrus.Logger, input []byte) Config {
+	logger := rootLogger.WithFields(logrus.Fields{"prefix": "config"})
+	return newConfigFromString(logger, input, "toml")
+}
+
+func newConfigFromString(logger *logrus.Entry, input []byte, cfgtype string) *config {
+	viper.SetConfigType(cfgtype)
+	viper.SetEnvPrefix("matterbridge")
+	viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_", "-", "_"))
+	viper.AutomaticEnv()
+
+	if err := viper.ReadConfig(bytes.NewBuffer(input)); err != nil {
+		logger.Fatalf("Failed to parse the configuration: %s", err)
+	}
+
+	cfg := &BridgeValues{}
+	if err := viper.Unmarshal(cfg); err != nil {
+		logger.Fatalf("Failed to load the configuration: %s", err)
+	}
+	return &config{
+		logger: logger,
+		v:      viper.GetViper(),
+		cv:     cfg,
 	}
-	mycfg := new(Config)
-	mycfg.v = viper.GetViper()
-	mycfg.ConfigValues = &cfg
-	return mycfg
 }
 
-func (c *Config) GetBool(key string) bool {
+func (c *config) BridgeValues() *BridgeValues {
-	c.RLock()
+	return c.cv
-	defer c.RUnlock()
-	//	log.Debugf("getting bool %s = %#v", key, c.v.GetBool(key))
-	return c.v.GetBool(key)
 }
 
-func (c *Config) GetInt(key string) int {
+func (c *config) Viper() *viper.Viper {
-	c.RLock()
+	return c.v
-	defer c.RUnlock()
-	//	log.Debugf("getting int %s = %d", key, c.v.GetInt(key))
-	return c.v.GetInt(key)
 }
 
-func (c *Config) GetString(key string) string {
+func (c *config) IsKeySet(key string) bool {
 	c.RLock()
 	defer c.RUnlock()
-	//	log.Debugf("getting String %s = %s", key, c.v.GetString(key))
+	return c.v.IsSet(key)
-	return c.v.GetString(key)
 }
 
-func (c *Config) GetStringSlice(key string) []string {
+func (c *config) GetBool(key string) (bool, bool) {
 	c.RLock()
 	defer c.RUnlock()
-	// log.Debugf("getting StringSlice %s = %#v", key, c.v.GetStringSlice(key))
+	return c.v.GetBool(key), c.v.IsSet(key)
-	return c.v.GetStringSlice(key)
 }
 
-func (c *Config) GetStringSlice2D(key string) [][]string {
+func (c *config) GetInt(key string) (int, bool) {
 	c.RLock()
 	defer c.RUnlock()
-	result := [][]string{}
+	return c.v.GetInt(key), c.v.IsSet(key)
-	if res, ok := c.v.Get(key).([]interface{}); ok {
+}
-		for _, entry := range res {
+
-			result2 := []string{}
+func (c *config) GetString(key string) (string, bool) {
-			for _, entry2 := range entry.([]interface{}) {
+	c.RLock()
-				result2 = append(result2, entry2.(string))
+	defer c.RUnlock()
-			}
+	return c.v.GetString(key), c.v.IsSet(key)
-			result = append(result, result2)
+}
+
+func (c *config) GetStringSlice(key string) ([]string, bool) {
+	c.RLock()
+	defer c.RUnlock()
+	return c.v.GetStringSlice(key), c.v.IsSet(key)
+}
+
+func (c *config) GetStringSlice2D(key string) ([][]string, bool) {
+	c.RLock()
+	defer c.RUnlock()
+
+	res, ok := c.v.Get(key).([]interface{})
+	if !ok {
+		return nil, false
+	}
+	var result [][]string
+	for _, entry := range res {
+		result2 := []string{}
+		for _, entry2 := range entry.([]interface{}) {
+			result2 = append(result2, entry2.(string))
 		}
-		return result
+		result = append(result, result2)
 	}
-	return result
+	return result, true
 }
 
 func GetIconURL(msg *Message, iconURL string) string {
@@ -284,3 +374,50 @@ func GetIconURL(msg *Message, iconURL string) string {
 	iconURL = strings.Replace(iconURL, "{PROTOCOL}", protocol, -1)
 	return iconURL
 }
+
+type TestConfig struct {
+	Config
+
+	Overrides map[string]interface{}
+}
+
+func (c *TestConfig) IsKeySet(key string) bool {
+	_, ok := c.Overrides[key]
+	return ok || c.Config.IsKeySet(key)
+}
+
+func (c *TestConfig) GetBool(key string) (bool, bool) {
+	val, ok := c.Overrides[key]
+	if ok {
+		return val.(bool), true
+	}
+	return c.Config.GetBool(key)
+}
+
+func (c *TestConfig) GetInt(key string) (int, bool) {
+	if val, ok := c.Overrides[key]; ok {
+		return val.(int), true
+	}
+	return c.Config.GetInt(key)
+}
+
+func (c *TestConfig) GetString(key string) (string, bool) {
+	if val, ok := c.Overrides[key]; ok {
+		return val.(string), true
+	}
+	return c.Config.GetString(key)
+}
+
+func (c *TestConfig) GetStringSlice(key string) ([]string, bool) {
+	if val, ok := c.Overrides[key]; ok {
+		return val.([]string), true
+	}
+	return c.Config.GetStringSlice(key)
+}
+
+func (c *TestConfig) GetStringSlice2D(key string) ([][]string, bool) {
+	if val, ok := c.Overrides[key]; ok {
+		return val.([][]string), true
+	}
+	return c.Config.GetStringSlice2D(key)
+}

bridge/discord/discord.go

@@ -2,35 +2,48 @@ package bdiscord
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
-	"regexp"
 	"strings"
 	"sync"
 
 	"github.com/42wim/matterbridge/bridge"
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/42wim/matterbridge/bridge/helper"
-	"github.com/bwmarrin/discordgo"
+	"github.com/matterbridge/discordgo"
 )
 
+const MessageLength = 1950
+
 type Bdiscord struct {
-	c              *discordgo.Session
-	Channels       []*discordgo.Channel
-	Nick           string
-	UseChannelID   bool
-	userMemberMap  map[string]*discordgo.Member
-	guildID        string
-	webhookID      string
-	webhookToken   string
-	channelInfoMap map[string]*config.ChannelInfo
-	sync.RWMutex
 	*bridge.Config
+
+	c *discordgo.Session
+
+	nick            string
+	userID          string
+	guildID         string
+	webhookID       string
+	webhookToken    string
+	canEditWebhooks bool
+
+	channelsMutex  sync.RWMutex
+	channels       []*discordgo.Channel
+	channelInfoMap map[string]*config.ChannelInfo
+
+	membersMutex  sync.RWMutex
+	userMemberMap map[string]*discordgo.Member
+	nickMemberMap map[string]*discordgo.Member
+	webhookCache  map[string]string
+	webhookMutex  sync.RWMutex
 }
 
 func New(cfg *bridge.Config) bridge.Bridger {
 	b := &Bdiscord{Config: cfg}
 	b.userMemberMap = make(map[string]*discordgo.Member)
+	b.nickMemberMap = make(map[string]*discordgo.Member)
 	b.channelInfoMap = make(map[string]*config.ChannelInfo)
+	b.webhookCache = make(map[string]string)
 	if b.GetString("WebhookURL") != "" {
 		b.Log.Debug("Configuring Discord Incoming Webhook")
 		b.webhookID, b.webhookToken = b.splitURL(b.GetString("WebhookURL"))
@@ -40,7 +53,8 @@ func New(cfg *bridge.Config) bridge.Bridger {
 
 func (b *Bdiscord) Connect() error {
 	var err error
-	var token string
+	var guildFound bool
+	token := b.GetString("Token")
 	b.Log.Info("Connecting")
 	if b.GetString("WebhookURL") == "" {
 		b.Log.Info("Connecting using token")
@@ -50,15 +64,24 @@ func (b *Bdiscord) Connect() error {
 	if !strings.HasPrefix(b.GetString("Token"), "Bot ") {
 		token = "Bot " + b.GetString("Token")
 	}
+	// if we have a User token, remove the `Bot` prefix
+	if strings.HasPrefix(b.GetString("Token"), "User ") {
+		token = strings.Replace(b.GetString("Token"), "User ", "", -1)
+	}
+
 	b.c, err = discordgo.New(token)
 	if err != nil {
 		return err
 	}
 	b.Log.Info("Connection succeeded")
 	b.c.AddHandler(b.messageCreate)
+	b.c.AddHandler(b.messageTyping)
 	b.c.AddHandler(b.memberUpdate)
 	b.c.AddHandler(b.messageUpdate)
 	b.c.AddHandler(b.messageDelete)
+	b.c.AddHandler(b.messageDeleteBulk)
+	b.c.AddHandler(b.memberAdd)
+	b.c.AddHandler(b.memberRemove)
 	err = b.c.Open()
 	if err != nil {
 		return err
@@ -71,18 +94,84 @@ func (b *Bdiscord) Connect() error {
 	if err != nil {
 		return err
 	}
-	b.Nick = userinfo.Username
+	serverName := strings.Replace(b.GetString("Server"), "ID:", "", -1)
+	b.nick = userinfo.Username
+	b.userID = userinfo.ID
+	b.channelsMutex.Lock()
 	for _, guild := range guilds {
-		if guild.Name == b.GetString("Server") {
+		if guild.Name == serverName || guild.ID == serverName {
-			b.Channels, err = b.c.GuildChannels(guild.ID)
+			b.channels, err = b.c.GuildChannels(guild.ID)
-			b.guildID = guild.ID
 			if err != nil {
-				return err
+				break
 			}
+			b.guildID = guild.ID
+			guildFound = true
 		}
 	}
-	for _, channel := range b.Channels {
+	b.channelsMutex.Unlock()
-		b.Log.Debugf("found channel %#v", channel)
+	if !guildFound {
+		msg := fmt.Sprintf("Server \"%s\" not found", b.GetString("Server"))
+		err = errors.New(msg)
+		b.Log.Error(msg)
+		b.Log.Info("Possible values:")
+		for _, guild := range guilds {
+			b.Log.Infof("Server=\"%s\" # Server name", guild.Name)
+			b.Log.Infof("Server=\"%s\" # Server ID", guild.ID)
+		}
+	}
+	if err != nil {
+		return err
+	}
+
+	b.channelsMutex.RLock()
+	if b.GetString("WebhookURL") == "" {
+		for _, channel := range b.channels {
+			b.Log.Debugf("found channel %#v", channel)
+		}
+	} else {
+		manageWebhooks := discordgo.PermissionManageWebhooks
+		var channelsDenied []string
+		for _, info := range b.Channels {
+			id := b.getChannelID(info.Name) // note(qaisjp): this readlocks channelsMutex
+			b.Log.Debugf("Verifying PermissionManageWebhooks for %s with ID %s", info.ID, id)
+
+			perms, permsErr := b.c.UserChannelPermissions(userinfo.ID, id)
+			if permsErr != nil {
+				b.Log.Warnf("Failed to check PermissionManageWebhooks in channel \"%s\": %s", info.Name, permsErr.Error())
+			} else if perms&manageWebhooks == manageWebhooks {
+				continue
+			}
+			channelsDenied = append(channelsDenied, fmt.Sprintf("%#v", info.Name))
+		}
+
+		b.canEditWebhooks = len(channelsDenied) == 0
+		if b.canEditWebhooks {
+			b.Log.Info("Can manage webhooks; will edit channel for global webhook on send")
+		} else {
+			b.Log.Warn("Can't manage webhooks; won't edit channel for global webhook on send")
+			b.Log.Warn("Can't manage webhooks in channels: ", strings.Join(channelsDenied, ", "))
+		}
+	}
+	b.channelsMutex.RUnlock()
+
+	// Obtaining guild members and initializing nickname mapping.
+	b.membersMutex.Lock()
+	defer b.membersMutex.Unlock()
+	members, err := b.c.GuildMembers(b.guildID, "", 1000)
+	if err != nil {
+		b.Log.Error("Error obtaining server members: ", err)
+		return err
+	}
+	for _, member := range members {
+		if member == nil {
+			b.Log.Warnf("Skipping missing information for a user.")
+			continue
+		}
+		b.userMemberMap[member.User.ID] = member
+		b.nickMemberMap[member.User.Username] = member
+		if member.Nick != "" {
+			b.nickMemberMap[member.Nick] = member
+		}
 	}
 	return nil
 }
@@ -92,70 +181,117 @@ func (b *Bdiscord) Disconnect() error {
 }
 
 func (b *Bdiscord) JoinChannel(channel config.ChannelInfo) error {
+	b.channelsMutex.Lock()
+	defer b.channelsMutex.Unlock()
+
 	b.channelInfoMap[channel.ID] = &channel
-	idcheck := strings.Split(channel.Name, "ID:")
-	if len(idcheck) > 1 {
-		b.UseChannelID = true
-	}
 	return nil
 }
 
 func (b *Bdiscord) Send(msg config.Message) (string, error) {
 	b.Log.Debugf("=> Receiving %#v", msg)
 
+	origMsgID := msg.ID
+
 	channelID := b.getChannelID(msg.Channel)
 	if channelID == "" {
 		return "", fmt.Errorf("Could not find channelID for %v", msg.Channel)
 	}
 
+	if msg.Event == config.EventUserTyping {
+		if b.GetBool("ShowUserTyping") {
+			err := b.c.ChannelTyping(channelID)
+			return "", err
+		}
+		return "", nil
+	}
+
 	// Make a action /me of the message
-	if msg.Event == config.EVENT_USER_ACTION {
+	if msg.Event == config.EventUserAction {
 		msg.Text = "_" + msg.Text + "_"
 	}
 
-	// use initial webhook
+	// use initial webhook configured for the entire Discord account
+	isGlobalWebhook := true
 	wID := b.webhookID
 	wToken := b.webhookToken
 
 	// check if have a channel specific webhook
+	b.channelsMutex.RLock()
 	if ci, ok := b.channelInfoMap[msg.Channel+b.Account]; ok {
 		if ci.Options.WebhookURL != "" {
 			wID, wToken = b.splitURL(ci.Options.WebhookURL)
+			isGlobalWebhook = false
 		}
 	}
+	b.channelsMutex.RUnlock()
 
 	// Use webhook to send the message
-	if wID != "" {
+	if wID != "" && msg.Event != config.EventMsgDelete {
 		// skip events
-		if msg.Event != "" {
+		if msg.Event != "" && msg.Event != config.EventUserAction && msg.Event != config.EventJoinLeave && msg.Event != config.EventTopicChange {
 			return "", nil
 		}
-		b.Log.Debugf("Broadcasting using Webhook")
+
-		for _, f := range msg.Extra["file"] {
+		// If we are editing a message, delete the old message
-			fi := f.(config.FileInfo)
+		if msg.ID != "" {
-			if fi.URL != "" {
+			msg.ID = b.getCacheID(msg.ID)
-				msg.Text += fi.URL + " "
+			b.Log.Debugf("Deleting edited webhook message")
+			err := b.c.ChannelMessageDelete(channelID, msg.ID)
+			if err != nil {
+				b.Log.Errorf("Could not delete edited webhook message: %s", err)
 			}
 		}
-		err := b.c.WebhookExecute(
+
-			wID,
+		b.Log.Debugf("Broadcasting using Webhook")
-			wToken,
+
-			true,
+		// skip empty messages
-			&discordgo.WebhookParams{
+		if msg.Text == "" && (msg.Extra == nil || len(msg.Extra["file"]) == 0) {
-				Content:   msg.Text,
+			b.Log.Debugf("Skipping empty message %#v", msg)
-				Username:  msg.Username,
+			return "", nil
-				AvatarURL: msg.Avatar,
+		}
-			})
+
-		return "", err
+		msg.Text = helper.ClipMessage(msg.Text, MessageLength)
+		msg.Text = b.replaceUserMentions(msg.Text)
+		// discord username must be [0..32] max
+		if len(msg.Username) > 32 {
+			msg.Username = msg.Username[0:32]
+		}
+		// if we have a global webhook for this Discord account, and permission
+		// to modify webhooks (previously verified), then set its channel to
+		// the message channel before using it
+		// TODO: this isn't necessary if the last message from this webhook was
+		// sent to the current channel
+		if isGlobalWebhook && b.canEditWebhooks {
+			b.Log.Debugf("Setting webhook channel to \"%s\"", msg.Channel)
+			_, err := b.c.WebhookEdit(wID, "", "", channelID)
+			if err != nil {
+				b.Log.Errorf("Could not set webhook channel: %s", err)
+				return "", err
+			}
+		}
+		b.Log.Debugf("Processing webhook sending for message %#v", msg)
+		msg, err := b.webhookSend(&msg, wID, wToken)
+		if err != nil {
+			b.Log.Errorf("Could not broadcast via webook for message %#v: %s", msg, err)
+			return "", err
+		}
+		if msg == nil {
+			return "", nil
+		}
+
+		b.updateCacheID(origMsgID, msg.ID)
+		return msg.ID, nil
 	}
 
 	b.Log.Debugf("Broadcasting using token (API)")
 
 	// Delete message
-	if msg.Event == config.EVENT_MSG_DELETE {
+	if msg.Event == config.EventMsgDelete {
 		if msg.ID == "" {
 			return "", nil
 		}
+		msg.ID = b.getCacheID(msg.ID)
 		err := b.c.ChannelMessageDelete(channelID, msg.ID)
 		return "", err
 	}
@@ -163,7 +299,10 @@ func (b *Bdiscord) Send(msg config.Message) (string, error) {
 	// Upload a file if it exists
 	if msg.Extra != nil {
 		for _, rmsg := range helper.HandleExtra(&msg, b.General) {
-			b.c.ChannelMessageSend(channelID, rmsg.Username+rmsg.Text)
+			rmsg.Text = helper.ClipMessage(rmsg.Text, MessageLength)
+			if _, err := b.c.ChannelMessageSend(channelID, rmsg.Username+rmsg.Text); err != nil {
+				b.Log.Errorf("Could not send message %#v: %s", rmsg, err)
+			}
 		}
 		// check if we have files to upload (from slack, telegram or mattermost)
 		if len(msg.Extra["file"]) > 0 {
@@ -171,6 +310,9 @@ func (b *Bdiscord) Send(msg config.Message) (string, error) {
 		}
 	}
 
+	msg.Text = helper.ClipMessage(msg.Text, MessageLength)
+	msg.Text = b.replaceUserMentions(msg.Text)
+
 	// Edit message
 	if msg.ID != "" {
 		_, err := b.c.ChannelMessageEdit(channelID, msg.ID, msg.Username+msg.Text)
@@ -182,198 +324,7 @@ func (b *Bdiscord) Send(msg config.Message) (string, error) {
 	if err != nil {
 		return "", err
 	}
-	return res.ID, err
+	return res.ID, nil
-}
-
-func (b *Bdiscord) messageDelete(s *discordgo.Session, m *discordgo.MessageDelete) {
-	rmsg := config.Message{Account: b.Account, ID: m.ID, Event: config.EVENT_MSG_DELETE, Text: config.EVENT_MSG_DELETE}
-	rmsg.Channel = b.getChannelName(m.ChannelID)
-	if b.UseChannelID {
-		rmsg.Channel = "ID:" + m.ChannelID
-	}
-	b.Log.Debugf("<= Sending message from %s to gateway", b.Account)
-	b.Log.Debugf("<= Message is %#v", rmsg)
-	b.Remote <- rmsg
-}
-
-func (b *Bdiscord) messageUpdate(s *discordgo.Session, m *discordgo.MessageUpdate) {
-	if b.GetBool("EditDisable") {
-		return
-	}
-	// only when message is actually edited
-	if m.Message.EditedTimestamp != "" {
-		b.Log.Debugf("Sending edit message")
-		m.Content = m.Content + b.GetString("EditSuffix")
-		b.messageCreate(s, (*discordgo.MessageCreate)(m))
-	}
-}
-
-func (b *Bdiscord) messageCreate(s *discordgo.Session, m *discordgo.MessageCreate) {
-	var err error
-
-	// not relay our own messages
-	if m.Author.Username == b.Nick {
-		return
-	}
-	// if using webhooks, do not relay if it's ours
-	if b.useWebhook() && m.Author.Bot && b.isWebhookID(m.Author.ID) {
-		return
-	}
-
-	// add the url of the attachments to content
-	if len(m.Attachments) > 0 {
-		for _, attach := range m.Attachments {
-			m.Content = m.Content + "\n" + attach.URL
-		}
-	}
-
-	rmsg := config.Message{Account: b.Account, Avatar: "https://cdn.discordapp.com/avatars/" + m.Author.ID + "/" + m.Author.Avatar + ".jpg", UserID: m.Author.ID, ID: m.ID}
-
-	if m.Content != "" {
-		b.Log.Debugf("== Receiving event %#v", m.Message)
-		m.Message.Content = b.stripCustomoji(m.Message.Content)
-		m.Message.Content = b.replaceChannelMentions(m.Message.Content)
-		rmsg.Text, err = m.ContentWithMoreMentionsReplaced(b.c)
-		if err != nil {
-			b.Log.Errorf("ContentWithMoreMentionsReplaced failed: %s", err)
-			rmsg.Text = m.ContentWithMentionsReplaced()
-		}
-	}
-
-	// set channel name
-	rmsg.Channel = b.getChannelName(m.ChannelID)
-	if b.UseChannelID {
-		rmsg.Channel = "ID:" + m.ChannelID
-	}
-
-	// set username
-	if !b.GetBool("UseUserName") {
-		rmsg.Username = b.getNick(m.Author)
-	} else {
-		rmsg.Username = m.Author.Username
-	}
-
-	// if we have embedded content add it to text
-	if b.GetBool("ShowEmbeds") && m.Message.Embeds != nil {
-		for _, embed := range m.Message.Embeds {
-			rmsg.Text = rmsg.Text + "embed: " + embed.Title + " - " + embed.Description + " - " + embed.URL + "\n"
-		}
-	}
-
-	// no empty messages
-	if rmsg.Text == "" {
-		return
-	}
-
-	// do we have a /me action
-	var ok bool
-	rmsg.Text, ok = b.replaceAction(rmsg.Text)
-	if ok {
-		rmsg.Event = config.EVENT_USER_ACTION
-	}
-
-	b.Log.Debugf("<= Sending message from %s on %s to gateway", m.Author.Username, b.Account)
-	b.Log.Debugf("<= Message is %#v", rmsg)
-	b.Remote <- rmsg
-}
-
-func (b *Bdiscord) memberUpdate(s *discordgo.Session, m *discordgo.GuildMemberUpdate) {
-	b.Lock()
-	if _, ok := b.userMemberMap[m.Member.User.ID]; ok {
-		b.Log.Debugf("%s: memberupdate: user %s (nick %s) changes nick to %s", b.Account, m.Member.User.Username, b.userMemberMap[m.Member.User.ID].Nick, m.Member.Nick)
-	}
-	b.userMemberMap[m.Member.User.ID] = m.Member
-	b.Unlock()
-}
-
-func (b *Bdiscord) getNick(user *discordgo.User) string {
-	var err error
-	b.Lock()
-	defer b.Unlock()
-	if _, ok := b.userMemberMap[user.ID]; ok {
-		if b.userMemberMap[user.ID] != nil {
-			if b.userMemberMap[user.ID].Nick != "" {
-				// only return if nick is set
-				return b.userMemberMap[user.ID].Nick
-			}
-			// otherwise return username
-			return user.Username
-		}
-	}
-	// if we didn't find nick, search for it
-	member, err := b.c.GuildMember(b.guildID, user.ID)
-	if err != nil {
-		return user.Username
-	}
-	b.userMemberMap[user.ID] = member
-	// only return if nick is set
-	if b.userMemberMap[user.ID].Nick != "" {
-		return b.userMemberMap[user.ID].Nick
-	}
-	return user.Username
-}
-
-func (b *Bdiscord) getChannelID(name string) string {
-	idcheck := strings.Split(name, "ID:")
-	if len(idcheck) > 1 {
-		return idcheck[1]
-	}
-	for _, channel := range b.Channels {
-		if channel.Name == name {
-			return channel.ID
-		}
-	}
-	return ""
-}
-
-func (b *Bdiscord) getChannelName(id string) string {
-	for _, channel := range b.Channels {
-		if channel.ID == id {
-			return channel.Name
-		}
-	}
-	return ""
-}
-
-func (b *Bdiscord) replaceChannelMentions(text string) string {
-	var err error
-	re := regexp.MustCompile("<#[0-9]+>")
-	text = re.ReplaceAllStringFunc(text, func(m string) string {
-		channel := b.getChannelName(m[2 : len(m)-1])
-		// if at first don't succeed, try again
-		if channel == "" {
-			b.Channels, err = b.c.GuildChannels(b.guildID)
-			if err != nil {
-				return "#unknownchannel"
-			}
-			channel = b.getChannelName(m[2 : len(m)-1])
-			return "#" + channel
-		}
-		return "#" + channel
-	})
-	return text
-}
-
-func (b *Bdiscord) replaceAction(text string) (string, bool) {
-	if strings.HasPrefix(text, "_") && strings.HasSuffix(text, "_") {
-		return strings.Replace(text, "_", "", -1), true
-	}
-	return text, false
-}
-
-func (b *Bdiscord) stripCustomoji(text string) string {
-	// <:doge:302803592035958784>
-	re := regexp.MustCompile("<(:.*?:)[0-9]+>")
-	return re.ReplaceAllString(text, `$1`)
-}
-
-// splitURL splits a webhookURL and returns the id and token
-func (b *Bdiscord) splitURL(url string) (string, string) {
-	webhookURLSplit := strings.Split(url, "/")
-	if len(webhookURLSplit) != 7 {
-		b.Log.Fatalf("%s is no correct discord WebhookURL", url)
-	}
-	return webhookURLSplit[len(webhookURLSplit)-2], webhookURLSplit[len(webhookURLSplit)-1]
 }
 
 // useWebhook returns true if we have a webhook defined somewhere
@@ -381,6 +332,10 @@ func (b *Bdiscord) useWebhook() bool {
 	if b.GetString("WebhookURL") != "" {
 		return true
 	}
+
+	b.channelsMutex.RLock()
+	defer b.channelsMutex.RUnlock()
+
 	for _, channel := range b.channelInfoMap {
 		if channel.Options.WebhookURL != "" {
 			return true
@@ -397,6 +352,10 @@ func (b *Bdiscord) isWebhookID(id string) bool {
 			return true
 		}
 	}
+
+	b.channelsMutex.RLock()
+	defer b.channelsMutex.RUnlock()
+
 	for _, channel := range b.channelInfoMap {
 		if channel.Options.WebhookURL != "" {
 			wID, _ := b.splitURL(channel.Options.WebhookURL)
@@ -413,12 +372,99 @@ func (b *Bdiscord) handleUploadFile(msg *config.Message, channelID string) (stri
 	var err error
 	for _, f := range msg.Extra["file"] {
 		fi := f.(config.FileInfo)
-		files := []*discordgo.File{}
+		file := discordgo.File{
-		files = append(files, &discordgo.File{fi.Name, "", bytes.NewReader(*fi.Data)})
+			Name:        fi.Name,
-		_, err = b.c.ChannelMessageSendComplex(channelID, &discordgo.MessageSend{Content: msg.Username + fi.Comment, Files: files})
+			ContentType: "",
+			Reader:      bytes.NewReader(*fi.Data),
+		}
+		m := discordgo.MessageSend{
+			Content: msg.Username + fi.Comment,
+			Files:   []*discordgo.File{&file},
+		}
+		_, err = b.c.ChannelMessageSendComplex(channelID, &m)
 		if err != nil {
-			return "", fmt.Errorf("file upload failed: %#v", err)
+			return "", fmt.Errorf("file upload failed: %s", err)
 		}
 	}
 	return "", nil
 }
+
+// webhookSend send one or more message via webhook, taking care of file
+// uploads (from slack, telegram or mattermost).
+// Returns messageID and error.
+func (b *Bdiscord) webhookSend(msg *config.Message, webhookID, token string) (*discordgo.Message, error) {
+	var (
+		res *discordgo.Message
+		err error
+	)
+
+	// If avatar is unset, check if UseLocalAvatar contains the message's
+	// account or protocol, and if so, try to find a local avatar
+	if msg.Avatar == "" {
+		for _, val := range b.GetStringSlice("UseLocalAvatar") {
+			if msg.Protocol == val || msg.Account == val {
+				if avatar := b.findAvatar(msg); avatar != "" {
+					msg.Avatar = avatar
+				}
+				break
+			}
+		}
+	}
+
+	// WebhookParams can have either `Content` or `File`.
+
+	// We can't send empty messages.
+	if msg.Text != "" {
+		res, err = b.c.WebhookExecute(
+			webhookID,
+			token,
+			true,
+			&discordgo.WebhookParams{
+				Content:   msg.Text,
+				Username:  msg.Username,
+				AvatarURL: msg.Avatar,
+			},
+		)
+		if err != nil {
+			b.Log.Errorf("Could not send text (%s) for message %#v: %s", msg.Text, msg, err)
+		}
+	}
+
+	if msg.Extra != nil {
+		for _, f := range msg.Extra["file"] {
+			fi := f.(config.FileInfo)
+			file := discordgo.File{
+				Name:        fi.Name,
+				ContentType: "",
+				Reader:      bytes.NewReader(*fi.Data),
+			}
+			content := ""
+			if msg.Text == "" {
+				content = fi.Comment
+			}
+			_, e2 := b.c.WebhookExecute(
+				webhookID,
+				token,
+				false,
+				&discordgo.WebhookParams{
+					Username:  msg.Username,
+					AvatarURL: msg.Avatar,
+					File:      &file,
+					Content:   content,
+				},
+			)
+			if e2 != nil {
+				b.Log.Errorf("Could not send file %#v for message %#v: %s", file, msg, e2)
+			}
+		}
+	}
+	return res, err
+}
+
+func (b *Bdiscord) findAvatar(m *config.Message) string {
+	member, err := b.getGuildMemberByNick(m.Username)
+	if err != nil {
+		return ""
+	}
+	return member.User.AvatarURL("")
+}

bridge/discord/handlers.go

@@ -0,0 +1,232 @@
+package bdiscord
+
+import (
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/matterbridge/discordgo"
+)
+
+func (b *Bdiscord) messageDelete(s *discordgo.Session, m *discordgo.MessageDelete) { //nolint:unparam
+	rmsg := config.Message{Account: b.Account, ID: m.ID, Event: config.EventMsgDelete, Text: config.EventMsgDelete}
+	rmsg.Channel = b.getChannelName(m.ChannelID)
+
+	b.Log.Debugf("<= Sending message from %s to gateway", b.Account)
+	b.Log.Debugf("<= Message is %#v", rmsg)
+	b.Remote <- rmsg
+}
+
+// TODO(qaisjp): if other bridges support bulk deletions, it could be fanned out centrally
+func (b *Bdiscord) messageDeleteBulk(s *discordgo.Session, m *discordgo.MessageDeleteBulk) { //nolint:unparam
+	for _, msgID := range m.Messages {
+		rmsg := config.Message{
+			Account: b.Account,
+			ID:      msgID,
+			Event:   config.EventMsgDelete,
+			Text:    config.EventMsgDelete,
+			Channel: b.getChannelName(m.ChannelID),
+		}
+
+		b.Log.Debugf("<= Sending message from %s to gateway", b.Account)
+		b.Log.Debugf("<= Message is %#v", rmsg)
+		b.Remote <- rmsg
+	}
+}
+
+func (b *Bdiscord) messageTyping(s *discordgo.Session, m *discordgo.TypingStart) {
+	if !b.GetBool("ShowUserTyping") {
+		return
+	}
+
+	// Ignore our own typing messages
+	if m.UserID == b.userID {
+		return
+	}
+
+	rmsg := config.Message{Account: b.Account, Event: config.EventUserTyping}
+	rmsg.Channel = b.getChannelName(m.ChannelID)
+	b.Remote <- rmsg
+}
+
+func (b *Bdiscord) messageUpdate(s *discordgo.Session, m *discordgo.MessageUpdate) { //nolint:unparam
+	if b.GetBool("EditDisable") {
+		return
+	}
+	// only when message is actually edited
+	if m.Message.EditedTimestamp != "" {
+		b.Log.Debugf("Sending edit message")
+		m.Content += b.GetString("EditSuffix")
+		msg := &discordgo.MessageCreate{
+			Message: m.Message,
+		}
+		b.messageCreate(s, msg)
+	}
+}
+
+func (b *Bdiscord) messageCreate(s *discordgo.Session, m *discordgo.MessageCreate) { //nolint:unparam
+	var err error
+
+	// not relay our own messages
+	if m.Author.Username == b.nick {
+		return
+	}
+	// if using webhooks, do not relay if it's ours
+	if b.useWebhook() && m.Author.Bot && b.isWebhookID(m.Author.ID) {
+		return
+	}
+
+	// add the url of the attachments to content
+	if len(m.Attachments) > 0 {
+		for _, attach := range m.Attachments {
+			m.Content = m.Content + "\n" + attach.URL
+		}
+	}
+
+	rmsg := config.Message{Account: b.Account, Avatar: "https://cdn.discordapp.com/avatars/" + m.Author.ID + "/" + m.Author.Avatar + ".jpg", UserID: m.Author.ID, ID: m.ID}
+
+	if m.Content != "" {
+		b.Log.Debugf("== Receiving event %#v", m.Message)
+		m.Message.Content = b.replaceChannelMentions(m.Message.Content)
+		rmsg.Text, err = m.ContentWithMoreMentionsReplaced(b.c)
+		if err != nil {
+			b.Log.Errorf("ContentWithMoreMentionsReplaced failed: %s", err)
+			rmsg.Text = m.ContentWithMentionsReplaced()
+		}
+	}
+
+	// set channel name
+	rmsg.Channel = b.getChannelName(m.ChannelID)
+
+	fromWebhook := m.WebhookID != ""
+	if !fromWebhook && !b.GetBool("UseUserName") {
+		rmsg.Username = b.getNick(m.Author, m.GuildID)
+	} else {
+		rmsg.Username = m.Author.Username
+		if !fromWebhook && b.GetBool("UseDiscriminator") {
+			rmsg.Username += "#" + m.Author.Discriminator
+		}
+	}
+
+	// if we have embedded content add it to text
+	if b.GetBool("ShowEmbeds") && m.Message.Embeds != nil {
+		for _, embed := range m.Message.Embeds {
+			rmsg.Text += handleEmbed(embed)
+		}
+	}
+
+	// no empty messages
+	if rmsg.Text == "" {
+		return
+	}
+
+	// do we have a /me action
+	var ok bool
+	rmsg.Text, ok = b.replaceAction(rmsg.Text)
+	if ok {
+		rmsg.Event = config.EventUserAction
+	}
+
+	// Replace emotes
+	rmsg.Text = replaceEmotes(rmsg.Text)
+
+	b.Log.Debugf("<= Sending message from %s on %s to gateway", m.Author.Username, b.Account)
+	b.Log.Debugf("<= Message is %#v", rmsg)
+	b.Remote <- rmsg
+}
+
+func (b *Bdiscord) memberUpdate(s *discordgo.Session, m *discordgo.GuildMemberUpdate) {
+	if m.Member == nil {
+		b.Log.Warnf("Received member update with no member information: %#v", m)
+	}
+
+	b.membersMutex.Lock()
+	defer b.membersMutex.Unlock()
+
+	if currMember, ok := b.userMemberMap[m.Member.User.ID]; ok {
+		b.Log.Debugf(
+			"%s: memberupdate: user %s (nick %s) changes nick to %s",
+			b.Account,
+			m.Member.User.Username,
+			b.userMemberMap[m.Member.User.ID].Nick,
+			m.Member.Nick,
+		)
+		delete(b.nickMemberMap, currMember.User.Username)
+		delete(b.nickMemberMap, currMember.Nick)
+		delete(b.userMemberMap, m.Member.User.ID)
+	}
+	b.userMemberMap[m.Member.User.ID] = m.Member
+	b.nickMemberMap[m.Member.User.Username] = m.Member
+	if m.Member.Nick != "" {
+		b.nickMemberMap[m.Member.Nick] = m.Member
+	}
+}
+
+func (b *Bdiscord) memberAdd(s *discordgo.Session, m *discordgo.GuildMemberAdd) {
+	if m.Member == nil {
+		b.Log.Warnf("Received member update with no member information: %#v", m)
+		return
+	}
+	username := m.Member.User.Username
+	if m.Member.Nick != "" {
+		username = m.Member.Nick
+	}
+
+	rmsg := config.Message{
+		Account:  b.Account,
+		Event:    config.EventJoinLeave,
+		Username: "system",
+		Text:     username + " joins",
+	}
+	b.Log.Debugf("<= Sending message from %s to gateway", b.Account)
+	b.Log.Debugf("<= Message is %#v", rmsg)
+	b.Remote <- rmsg
+}
+
+func (b *Bdiscord) memberRemove(s *discordgo.Session, m *discordgo.GuildMemberRemove) {
+	if m.Member == nil {
+		b.Log.Warnf("Received member update with no member information: %#v", m)
+		return
+	}
+	username := m.Member.User.Username
+	if m.Member.Nick != "" {
+		username = m.Member.Nick
+	}
+
+	rmsg := config.Message{
+		Account:  b.Account,
+		Event:    config.EventJoinLeave,
+		Username: "system",
+		Text:     username + " leaves",
+	}
+	b.Log.Debugf("<= Sending message from %s to gateway", b.Account)
+	b.Log.Debugf("<= Message is %#v", rmsg)
+	b.Remote <- rmsg
+}
+
+func handleEmbed(embed *discordgo.MessageEmbed) string {
+	var t []string
+	var result string
+
+	t = append(t, embed.Title)
+	t = append(t, embed.Description)
+	t = append(t, embed.URL)
+
+	i := 0
+	for _, e := range t {
+		if e == "" {
+			continue
+		}
+
+		i++
+		if i == 1 {
+			result += " embed: " + e
+			continue
+		}
+
+		result += " - " + e
+	}
+
+	if result != "" {
+		result += "\n"
+	}
+
+	return result
+}

bridge/discord/handlers_test.go

@@ -0,0 +1,58 @@
+package bdiscord
+
+import (
+	"testing"
+
+	"github.com/matterbridge/discordgo"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestHandleEmbed(t *testing.T) {
+	testcases := map[string]struct {
+		embed  *discordgo.MessageEmbed
+		result string
+	}{
+		"allempty": {
+			embed:  &discordgo.MessageEmbed{},
+			result: "",
+		},
+		"one": {
+			embed: &discordgo.MessageEmbed{
+				Title: "blah",
+			},
+			result: " embed: blah\n",
+		},
+		"two": {
+			embed: &discordgo.MessageEmbed{
+				Title:       "blah",
+				Description: "blah2",
+			},
+			result: " embed: blah - blah2\n",
+		},
+		"three": {
+			embed: &discordgo.MessageEmbed{
+				Title:       "blah",
+				Description: "blah2",
+				URL:         "blah3",
+			},
+			result: " embed: blah - blah2 - blah3\n",
+		},
+		"twob": {
+			embed: &discordgo.MessageEmbed{
+				Description: "blah2",
+				URL:         "blah3",
+			},
+			result: " embed: blah2 - blah3\n",
+		},
+		"oneb": {
+			embed: &discordgo.MessageEmbed{
+				URL: "blah3",
+			},
+			result: " embed: blah3\n",
+		},
+	}
+
+	for name, tc := range testcases {
+		assert.Equalf(t, tc.result, handleEmbed(tc.embed), "Testcases %s", name)
+	}
+}

bridge/discord/helpers.go

@@ -0,0 +1,279 @@
+package bdiscord
+
+import (
+	"errors"
+	"regexp"
+	"strings"
+	"unicode"
+
+	"github.com/matterbridge/discordgo"
+)
+
+func (b *Bdiscord) getNick(user *discordgo.User, guildID string) string {
+	b.membersMutex.RLock()
+	defer b.membersMutex.RUnlock()
+
+	if member, ok := b.userMemberMap[user.ID]; ok {
+		if member.Nick != "" {
+			// Only return if nick is set.
+			return member.Nick
+		}
+		// Otherwise return username.
+		return user.Username
+	}
+
+	// If we didn't find nick, search for it.
+	member, err := b.c.GuildMember(guildID, user.ID)
+	if err != nil {
+		b.Log.Warnf("Failed to fetch information for member %#v on guild %#v: %s", user, guildID, err)
+		return user.Username
+	} else if member == nil {
+		b.Log.Warnf("Got no information for member %#v", user)
+		return user.Username
+	}
+	b.userMemberMap[user.ID] = member
+	b.nickMemberMap[member.User.Username] = member
+	if member.Nick != "" {
+		b.nickMemberMap[member.Nick] = member
+		return member.Nick
+	}
+	return user.Username
+}
+
+func (b *Bdiscord) getGuildMemberByNick(nick string) (*discordgo.Member, error) {
+	b.membersMutex.RLock()
+	defer b.membersMutex.RUnlock()
+
+	if member, ok := b.nickMemberMap[nick]; ok {
+		return member, nil
+	}
+	return nil, errors.New("Couldn't find guild member with nick " + nick) // This will most likely get ignored by the caller
+}
+
+func (b *Bdiscord) getChannelID(name string) string {
+	if strings.Contains(name, "/") {
+		return b.getCategoryChannelID(name)
+	}
+	b.channelsMutex.RLock()
+	defer b.channelsMutex.RUnlock()
+
+	idcheck := strings.Split(name, "ID:")
+	if len(idcheck) > 1 {
+		return idcheck[1]
+	}
+	for _, channel := range b.channels {
+		if channel.Name == name && channel.Type == discordgo.ChannelTypeGuildText {
+			return channel.ID
+		}
+	}
+	return ""
+}
+
+func (b *Bdiscord) getCategoryChannelID(name string) string {
+	b.channelsMutex.RLock()
+	defer b.channelsMutex.RUnlock()
+	res := strings.Split(name, "/")
+	// shouldn't happen because function should be only called from getChannelID
+	if len(res) != 2 {
+		return ""
+	}
+	catName, chanName := res[0], res[1]
+	for _, channel := range b.channels {
+		// if we have a parentID, lookup the name of that parent (category)
+		// and if it matches return it
+		if channel.Name == chanName && channel.ParentID != "" {
+			for _, cat := range b.channels {
+				if cat.ID == channel.ParentID && cat.Name == catName {
+					return channel.ID
+				}
+			}
+		}
+	}
+	return ""
+}
+
+func (b *Bdiscord) getChannelName(id string) string {
+	b.channelsMutex.RLock()
+	defer b.channelsMutex.RUnlock()
+
+	for _, c := range b.channelInfoMap {
+		if c.Name == "ID:"+id {
+			// if we have ID: specified in our gateway configuration return this
+			return c.Name
+		}
+	}
+
+	for _, channel := range b.channels {
+		if channel.ID == id {
+			return b.getCategoryChannelName(channel.Name, channel.ParentID)
+		}
+	}
+	return ""
+}
+
+func (b *Bdiscord) getCategoryChannelName(name, parentID string) string {
+	var usesCat bool
+	// do we have a category configuration in the channel config
+	for _, c := range b.channelInfoMap {
+		if strings.Contains(c.Name, "/") {
+			usesCat = true
+			break
+		}
+	}
+	// configuration without category, return the normal channel name
+	if !usesCat {
+		return name
+	}
+	// create a category/channel response
+	for _, c := range b.channels {
+		if c.ID == parentID {
+			name = c.Name + "/" + name
+		}
+	}
+	return name
+}
+
+var (
+	// See https://discordapp.com/developers/docs/reference#message-formatting.
+	channelMentionRE = regexp.MustCompile("<#[0-9]+>")
+	userMentionRE    = regexp.MustCompile("@[^@\n]{1,32}")
+	emoteRE          = regexp.MustCompile(`<a?(:\w+:)\d+>`)
+)
+
+func (b *Bdiscord) replaceChannelMentions(text string) string {
+	replaceChannelMentionFunc := func(match string) string {
+		channelID := match[2 : len(match)-1]
+		channelName := b.getChannelName(channelID)
+
+		// If we don't have the channel refresh our list.
+		if channelName == "" {
+			var err error
+			b.channels, err = b.c.GuildChannels(b.guildID)
+			if err != nil {
+				return "#unknownchannel"
+			}
+			channelName = b.getChannelName(channelID)
+		}
+		return "#" + channelName
+	}
+	return channelMentionRE.ReplaceAllStringFunc(text, replaceChannelMentionFunc)
+}
+
+func (b *Bdiscord) replaceUserMentions(text string) string {
+	replaceUserMentionFunc := func(match string) string {
+		var (
+			err      error
+			member   *discordgo.Member
+			username string
+		)
+
+		usernames := enumerateUsernames(match[1:])
+		for _, username = range usernames {
+			b.Log.Debugf("Testing mention: '%s'", username)
+			member, err = b.getGuildMemberByNick(username)
+			if err == nil {
+				break
+			}
+		}
+		if member == nil {
+			return match
+		}
+		return strings.Replace(match, "@"+username, member.User.Mention(), 1)
+	}
+	return userMentionRE.ReplaceAllStringFunc(text, replaceUserMentionFunc)
+}
+
+func replaceEmotes(text string) string {
+	return emoteRE.ReplaceAllString(text, "$1")
+}
+
+func (b *Bdiscord) replaceAction(text string) (string, bool) {
+	length := len(text)
+	if length > 1 && text[0] == '_' && text[length-1] == '_' {
+		return text[1 : length-1], true
+	}
+	return text, false
+}
+
+// splitURL splits a webhookURL and returns the ID and token.
+func (b *Bdiscord) splitURL(url string) (string, string) {
+	const (
+		expectedWebhookSplitCount = 7
+		webhookIdxID              = 5
+		webhookIdxToken           = 6
+	)
+	webhookURLSplit := strings.Split(url, "/")
+	if len(webhookURLSplit) != expectedWebhookSplitCount {
+		b.Log.Fatalf("%s is no correct discord WebhookURL", url)
+	}
+	return webhookURLSplit[webhookIdxID], webhookURLSplit[webhookIdxToken]
+}
+
+// getcacheID tries to find a corresponding msgID in the webhook cache.
+// if not found returns the original request.
+func (b *Bdiscord) getCacheID(msgID string) string {
+	b.webhookMutex.RLock()
+	defer b.webhookMutex.RUnlock()
+	for k, v := range b.webhookCache {
+		if msgID == k {
+			return v
+		}
+	}
+	return msgID
+}
+
+// updateCacheID updates the cache so that the newID takes the place of
+// the original ID. This is used for edit/deletes in combination with webhooks
+// as editing a message via webhook means deleting the message and creating a
+// new message (with a new ID). This ID needs to be set instead of the original ID
+func (b *Bdiscord) updateCacheID(origID, newID string) {
+	b.webhookMutex.Lock()
+	match := false
+	for k, v := range b.webhookCache {
+		if v == origID {
+			delete(b.webhookCache, k)
+			b.webhookCache[origID] = newID
+			match = true
+			continue
+		}
+	}
+	if !match && origID != "" {
+		b.webhookCache[origID] = newID
+	}
+	b.webhookMutex.Unlock()
+}
+
+func enumerateUsernames(s string) []string {
+	onlySpace := true
+	for _, r := range s {
+		if !unicode.IsSpace(r) {
+			onlySpace = false
+			break
+		}
+	}
+	if onlySpace {
+		return nil
+	}
+
+	var username, endSpace string
+	var usernames []string
+	skippingSpace := true
+	for _, r := range s {
+		if unicode.IsSpace(r) {
+			if !skippingSpace {
+				usernames = append(usernames, username)
+				skippingSpace = true
+			}
+			endSpace += string(r)
+			username += string(r)
+		} else {
+			endSpace = ""
+			username += string(r)
+			skippingSpace = false
+		}
+	}
+	if endSpace == "" {
+		usernames = append(usernames, username)
+	}
+	return usernames
+}

bridge/discord/helpers_test.go

@@ -0,0 +1,46 @@
+package bdiscord
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestEnumerateUsernames(t *testing.T) {
+	testcases := map[string]struct {
+		match             string
+		expectedUsernames []string
+	}{
+		"only space": {
+			match:             "  \t\n \t",
+			expectedUsernames: nil,
+		},
+		"single word": {
+			match:             "veni",
+			expectedUsernames: []string{"veni"},
+		},
+		"single word with preceeding space": {
+			match:             " vidi",
+			expectedUsernames: []string{" vidi"},
+		},
+		"single word with suffixed space": {
+			match:             "vici ",
+			expectedUsernames: []string{"vici"},
+		},
+		"multi-word with varying whitespace": {
+			match: "just me  and\tmy friends \t",
+			expectedUsernames: []string{
+				"just",
+				"just me",
+				"just me  and",
+				"just me  and\tmy",
+				"just me  and\tmy friends",
+			},
+		},
+	}
+
+	for testname, testcase := range testcases {
+		foundUsernames := enumerateUsernames(testcase.match)
+		assert.Equalf(t, testcase.expectedUsernames, foundUsernames, "Should have found the expected usernames for testcase %s", testname)
+	}
+}

bridge/gitter/gitter.go

@@ -77,7 +77,7 @@ func (b *Bgitter) JoinChannel(channel config.ChannelInfo) error {
 						Account: b.Account, Avatar: b.getAvatar(ev.Message.From.Username), UserID: ev.Message.From.ID,
 						ID: ev.Message.ID}
 					if strings.HasPrefix(ev.Message.Text, "@"+ev.Message.From.Username) {
-						rmsg.Event = config.EVENT_USER_ACTION
+						rmsg.Event = config.EventUserAction
 						rmsg.Text = strings.Replace(rmsg.Text, "@"+ev.Message.From.Username+" ", "", -1)
 					}
 					b.Log.Debugf("<= Message is %#v", rmsg)
@@ -100,7 +100,7 @@ func (b *Bgitter) Send(msg config.Message) (string, error) {
 	}
 
 	// Delete message
-	if msg.Event == config.EVENT_MSG_DELETE {
+	if msg.Event == config.EventMsgDelete {
 		if msg.ID == "" {
 			return "", nil
 		}

bridge/helper/helper.go

@@ -3,20 +3,32 @@ package helper
 import (
 	"bytes"
 	"fmt"
+	"image/png"
 	"io"
+	"io/ioutil"
 	"net/http"
+	"os"
+	"os/exec"
 	"regexp"
 	"strings"
 	"time"
+	"unicode/utf8"
+
+	"golang.org/x/image/webp"
 
 	"github.com/42wim/matterbridge/bridge/config"
-	log "github.com/sirupsen/logrus"
+	"github.com/gomarkdown/markdown"
+	"github.com/gomarkdown/markdown/html"
+	"github.com/gomarkdown/markdown/parser"
+	"github.com/sirupsen/logrus"
 )
 
+// DownloadFile downloads the given non-authenticated URL.
 func DownloadFile(url string) (*[]byte, error) {
 	return DownloadFileAuth(url, "")
 }
 
+// DownloadFileAuth downloads the given URL using the specified authentication token.
 func DownloadFileAuth(url string, auth string) (*[]byte, error) {
 	var buf bytes.Buffer
 	client := &http.Client{
@@ -39,33 +51,62 @@ func DownloadFileAuth(url string, auth string) (*[]byte, error) {
 	return &data, nil
 }
 
-func SplitStringLength(input string, length int) string {
+// GetSubLines splits messages in newline-delimited lines. If maxLineLength is
-	a := []rune(input)
+// specified as non-zero GetSubLines will also clip long lines to the maximum
-	str := ""
+// length and insert a warning marker that the line was clipped.
-	for i, r := range a {
+//
-		str = str + string(r)
+// TODO: The current implementation has the inconvenient that it disregards
-		if i > 0 && (i+1)%length == 0 {
+// word boundaries when splitting but this is hard to solve without potentially
-			str += "\n"
+// breaking formatting and other stylistic effects.
+func GetSubLines(message string, maxLineLength int) []string {
+	const clippingMessage = " <clipped message>"
+
+	var lines []string
+	for _, line := range strings.Split(strings.TrimSpace(message), "\n") {
+		if maxLineLength == 0 || len([]byte(line)) <= maxLineLength {
+			lines = append(lines, line)
+			continue
 		}
+
+		// !!! WARNING !!!
+		// Before touching the splitting logic below please ensure that you PROPERLY
+		// understand how strings, runes and range loops over strings work in Go.
+		// A good place to start is to read https://blog.golang.org/strings. :-)
+		var splitStart int
+		var startOfPreviousRune int
+		for i := range line {
+			if i-splitStart > maxLineLength-len([]byte(clippingMessage)) {
+				lines = append(lines, line[splitStart:startOfPreviousRune]+clippingMessage)
+				splitStart = startOfPreviousRune
+			}
+			startOfPreviousRune = i
+		}
+		// This last append is safe to do without looking at the remaining byte-length
+		// as we assume that the byte-length of the last rune will never exceed that of
+		// the byte-length of the clipping message.
+		lines = append(lines, line[splitStart:])
 	}
-	return str
+	return lines
 }
 
-// handle all the stuff we put into extra
+// HandleExtra manages the supplementary details stored inside a message's 'Extra' field map.
 func HandleExtra(msg *config.Message, general *config.Protocol) []config.Message {
 	extra := msg.Extra
 	rmsg := []config.Message{}
-	if len(extra[config.EVENT_FILE_FAILURE_SIZE]) > 0 {
+	for _, f := range extra[config.EventFileFailureSize] {
-		for _, f := range extra[config.EVENT_FILE_FAILURE_SIZE] {
+		fi := f.(config.FileInfo)
-			fi := f.(config.FileInfo)
+		text := fmt.Sprintf("file %s too big to download (%#v > allowed size: %#v)", fi.Name, fi.Size, general.MediaDownloadSize)
-			text := fmt.Sprintf("file %s too big to download (%#v > allowed size: %#v)", fi.Name, fi.Size, general.MediaDownloadSize)
+		rmsg = append(rmsg, config.Message{
-			rmsg = append(rmsg, config.Message{Text: text, Username: "<system> ", Channel: msg.Channel})
+			Text:     text,
-		}
+			Username: "<system> ",
-		return rmsg
+			Channel:  msg.Channel,
+			Account:  msg.Account,
+		})
 	}
 	return rmsg
 }
 
+// GetAvatar constructs a URL for a given user-avatar if it is available in the cache.
 func GetAvatar(av map[string]string, userid string, general *config.Protocol) string {
 	if sha, ok := av[userid]; ok {
 		return general.MediaServerDownload + "/" + sha + "/" + userid + ".png"
@@ -73,13 +114,15 @@ func GetAvatar(av map[string]string, userid string, general *config.Protocol) st
 	return ""
 }
 
-func HandleDownloadSize(flog *log.Entry, msg *config.Message, name string, size int64, general *config.Protocol) error {
+// HandleDownloadSize checks a specified filename against the configured download blacklist
+// and checks a specified file-size against the configure limit.
+func HandleDownloadSize(logger *logrus.Entry, msg *config.Message, name string, size int64, general *config.Protocol) error {
 	// check blacklist here
 	for _, entry := range general.MediaDownloadBlackList {
 		if entry != "" {
 			re, err := regexp.Compile(entry)
 			if err != nil {
-				flog.Errorf("incorrect regexp %s for %s", entry, msg.Account)
+				logger.Errorf("incorrect regexp %s for %s", entry, msg.Account)
 				continue
 			}
 			if re.MatchString(name) {
@@ -87,31 +130,129 @@ func HandleDownloadSize(flog *log.Entry, msg *config.Message, name string, size
 			}
 		}
 	}
-	flog.Debugf("Trying to download %#v with size %#v", name, size)
+	logger.Debugf("Trying to download %#v with size %#v", name, size)
 	if int(size) > general.MediaDownloadSize {
-		msg.Event = config.EVENT_FILE_FAILURE_SIZE
+		msg.Event = config.EventFileFailureSize
-		msg.Extra[msg.Event] = append(msg.Extra[msg.Event], config.FileInfo{Name: name, Comment: msg.Text, Size: size})
+		msg.Extra[msg.Event] = append(msg.Extra[msg.Event], config.FileInfo{
+			Name:    name,
+			Comment: msg.Text,
+			Size:    size,
+		})
 		return fmt.Errorf("File %#v to large to download (%#v). MediaDownloadSize is %#v", name, size, general.MediaDownloadSize)
 	}
 	return nil
 }
 
-func HandleDownloadData(flog *log.Entry, msg *config.Message, name, comment, url string, data *[]byte, general *config.Protocol) {
+// HandleDownloadData adds the data for a remote file into a Matterbridge gateway message.
+func HandleDownloadData(logger *logrus.Entry, msg *config.Message, name, comment, url string, data *[]byte, general *config.Protocol) {
 	var avatar bool
-	flog.Debugf("Download OK %#v %#v", name, len(*data))
+	logger.Debugf("Download OK %#v %#v", name, len(*data))
-	if msg.Event == config.EVENT_AVATAR_DOWNLOAD {
+	if msg.Event == config.EventAvatarDownload {
 		avatar = true
 	}
-	msg.Extra["file"] = append(msg.Extra["file"], config.FileInfo{Name: name, Data: data, URL: url, Comment: comment, Avatar: avatar})
+	msg.Extra["file"] = append(msg.Extra["file"], config.FileInfo{
+		Name:    name,
+		Data:    data,
+		URL:     url,
+		Comment: comment,
+		Avatar:  avatar,
+	})
 }
 
+var emptyLineMatcher = regexp.MustCompile("\n+")
+
+// RemoveEmptyNewLines collapses consecutive newline characters into a single one and
+// trims any preceding or trailing newline characters as well.
 func RemoveEmptyNewLines(msg string) string {
-	lines := ""
+	return emptyLineMatcher.ReplaceAllString(strings.Trim(msg, "\n"), "\n")
-	for _, line := range strings.Split(msg, "\n") {
+}
-		if line != "" {
+
-			lines += line + "\n"
+// ClipMessage trims a message to the specified length if it exceeds it and adds a warning
-		}
+// to the message in case it does so.
-	}
+func ClipMessage(text string, length int) string {
-	lines = strings.TrimRight(lines, "\n")
+	const clippingMessage = " <clipped message>"
-	return lines
+	if len(text) > length {
+		text = text[:length-len(clippingMessage)]
+		if r, size := utf8.DecodeLastRuneInString(text); r == utf8.RuneError {
+			text = text[:len(text)-size]
+		}
+		text += clippingMessage
+	}
+	return text
+}
+
+// ParseMarkdown takes in an input string as markdown and parses it to html
+func ParseMarkdown(input string) string {
+	extensions := parser.HardLineBreak | parser.NoIntraEmphasis
+	markdownParser := parser.NewWithExtensions(extensions)
+	renderer := html.NewRenderer(html.RendererOptions{
+		Flags: 0,
+	})
+	parsedMarkdown := markdown.ToHTML([]byte(input), markdownParser, renderer)
+	res := string(parsedMarkdown)
+	res = strings.TrimPrefix(res, "<p>")
+	res = strings.TrimSuffix(res, "</p>\n")
+	return res
+}
+
+// ConvertWebPToPNG converts input data (which should be WebP format) to PNG format
+func ConvertWebPToPNG(data *[]byte) error {
+	r := bytes.NewReader(*data)
+	m, err := webp.Decode(r)
+	if err != nil {
+		return err
+	}
+	var output []byte
+	w := bytes.NewBuffer(output)
+	if err := png.Encode(w, m); err != nil {
+		return err
+	}
+	*data = w.Bytes()
+	return nil
+}
+
+// CanConvertTgsToX Checks whether the external command necessary for ConvertTgsToX works.
+func CanConvertTgsToX() error {
+	// We depend on the fact that `lottie_convert.py --help` has exit status 0.
+	// Hyrum's Law predicted this, and Murphy's Law predicts that this will break eventually.
+	// However, there is no alternative like `lottie_convert.py --is-properly-installed`
+	cmd := exec.Command("lottie_convert.py", "--help")
+	return cmd.Run()
+}
+
+// ConvertTgsToWebP convert input data (which should be tgs format) to WebP format
+// This relies on an external command, which is ugly, but works.
+func ConvertTgsToX(data *[]byte, outputFormat string, logger *logrus.Entry) error {
+	// lottie can't handle input from a pipe, so write to a temporary file:
+	tmpFile, err := ioutil.TempFile(os.TempDir(), "matterbridge-lottie-*.tgs")
+	if err != nil {
+		return err
+	}
+	tmpFileName := tmpFile.Name()
+	defer func() {
+		if removeErr := os.Remove(tmpFileName); removeErr != nil {
+			logger.Errorf("Could not delete temporary file %s: %v", tmpFileName, removeErr)
+		}
+	}()
+
+	if _, writeErr := tmpFile.Write(*data); writeErr != nil {
+		return writeErr
+	}
+	// Must close before calling lottie to avoid data races:
+	if closeErr := tmpFile.Close(); closeErr != nil {
+		return closeErr
+	}
+
+	// Call lottie to transform:
+	cmd := exec.Command("lottie_convert.py", "--input-format", "lottie", "--output-format", outputFormat, tmpFileName, "/dev/stdout")
+	cmd.Stderr = nil
+	// NB: lottie writes progress into to stderr in all cases.
+	stdout, stderr := cmd.Output()
+	if stderr != nil {
+		// 'stderr' already contains some parts of Stderr, because it was set to 'nil'.
+		return stderr
+	}
+
+	*data = stdout
+	return nil
 }

bridge/helper/helper_test.go

@@ -0,0 +1,126 @@
+package helper
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+const testLineLength = 64
+
+var (
+	lineSplittingTestCases = map[string]struct {
+		input          string
+		splitOutput    []string
+		nonSplitOutput []string
+	}{
+		"Short single-line message": {
+			input:          "short",
+			splitOutput:    []string{"short"},
+			nonSplitOutput: []string{"short"},
+		},
+		"Long single-line message": {
+			input: "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.",
+			splitOutput: []string{
+				"Lorem ipsum dolor sit amet, consectetur adipis <clipped message>",
+				"cing elit, sed do eiusmod tempor incididunt ut <clipped message>",
+				" labore et dolore magna aliqua.",
+			},
+			nonSplitOutput: []string{"Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua."},
+		},
+		"Short multi-line message": {
+			input: "I\ncan't\nget\nno\nsatisfaction!",
+			splitOutput: []string{
+				"I",
+				"can't",
+				"get",
+				"no",
+				"satisfaction!",
+			},
+			nonSplitOutput: []string{
+				"I",
+				"can't",
+				"get",
+				"no",
+				"satisfaction!",
+			},
+		},
+		"Long multi-line message": {
+			input: "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.\n" +
+				"Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.\n" +
+				"Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.\n" +
+				"Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.",
+			splitOutput: []string{
+				"Lorem ipsum dolor sit amet, consectetur adipis <clipped message>",
+				"cing elit, sed do eiusmod tempor incididunt ut <clipped message>",
+				" labore et dolore magna aliqua.",
+				"Ut enim ad minim veniam, quis nostrud exercita <clipped message>",
+				"tion ullamco laboris nisi ut aliquip ex ea com <clipped message>",
+				"modo consequat.",
+				"Duis aute irure dolor in reprehenderit in volu <clipped message>",
+				"ptate velit esse cillum dolore eu fugiat nulla <clipped message>",
+				" pariatur.",
+				"Excepteur sint occaecat cupidatat non proident <clipped message>",
+				", sunt in culpa qui officia deserunt mollit an <clipped message>",
+				"im id est laborum.",
+			},
+			nonSplitOutput: []string{
+				"Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.",
+				"Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.",
+				"Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.",
+				"Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.",
+			},
+		},
+		"Message ending with new-line.": {
+			input:          "Newline ending\n",
+			splitOutput:    []string{"Newline ending"},
+			nonSplitOutput: []string{"Newline ending"},
+		},
+		"Long message containing UTF-8 multi-byte runes": {
+			input: "不布人個我此而及單石業喜資富下我河下日沒一我臺空達的常景便物沒為……子大我別名解成？生賣的全直黑，我自我結毛分洲了世當，是政福那是東；斯說",
+			splitOutput: []string{
+				"不布人個我此而及單石業喜資富下 <clipped message>",
+				"我河下日沒一我臺空達的常景便物 <clipped message>",
+				"沒為……子大我別名解成？生賣的 <clipped message>",
+				"全直黑，我自我結毛分洲了世當， <clipped message>",
+				"是政福那是東；斯說",
+			},
+			nonSplitOutput: []string{"不布人個我此而及單石業喜資富下我河下日沒一我臺空達的常景便物沒為……子大我別名解成？生賣的全直黑，我自我結毛分洲了世當，是政福那是東；斯說"},
+		},
+	}
+)
+
+func TestGetSubLines(t *testing.T) {
+	for testname, testcase := range lineSplittingTestCases {
+		splitLines := GetSubLines(testcase.input, testLineLength)
+		assert.Equalf(t, testcase.splitOutput, splitLines, "'%s' testcase should give expected lines with splitting.", testname)
+		for _, splitLine := range splitLines {
+			byteLength := len([]byte(splitLine))
+			assert.True(t, byteLength <= testLineLength, "Splitted line '%s' of testcase '%s' should not exceed the maximum byte-length (%d vs. %d).", splitLine, testcase, byteLength, testLineLength)
+		}
+
+		nonSplitLines := GetSubLines(testcase.input, 0)
+		assert.Equalf(t, testcase.nonSplitOutput, nonSplitLines, "'%s' testcase should give expected lines without splitting.", testname)
+	}
+}
+
+func TestConvertWebPToPNG(t *testing.T) {
+	if os.Getenv("LOCAL_TEST") == "" {
+		t.Skip()
+	}
+	input, err := ioutil.ReadFile("test.webp")
+	if err != nil {
+		t.Fail()
+	}
+	d := &input
+	err = ConvertWebPToPNG(d)
+	if err != nil {
+		t.Fail()
+	}
+	err = ioutil.WriteFile("test.png", *d, 0644)
+	if err != nil {
+		t.Fail()
+	}
+}

bridge/irc/handlers.go

@@ -0,0 +1,238 @@
+package birc
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/42wim/matterbridge/bridge/helper"
+	"github.com/lrstanley/girc"
+	"github.com/missdeer/golib/ic"
+	"github.com/paulrosania/go-charset/charset"
+	"github.com/saintfish/chardet"
+
+	// We need to import the 'data' package as an implicit dependency.
+	// See: https://godoc.org/github.com/paulrosania/go-charset/charset
+	_ "github.com/paulrosania/go-charset/data"
+)
+
+func (b *Birc) handleCharset(msg *config.Message) error {
+	if b.GetString("Charset") != "" {
+		switch b.GetString("Charset") {
+		case "gbk", "gb18030", "gb2312", "big5", "euc-kr", "euc-jp", "shift-jis", "iso-2022-jp":
+			msg.Text = ic.ConvertString("utf-8", b.GetString("Charset"), msg.Text)
+		default:
+			buf := new(bytes.Buffer)
+			w, err := charset.NewWriter(b.GetString("Charset"), buf)
+			if err != nil {
+				b.Log.Errorf("charset from utf-8 conversion failed: %s", err)
+				return err
+			}
+			fmt.Fprint(w, msg.Text)
+			w.Close()
+			msg.Text = buf.String()
+		}
+	}
+	return nil
+}
+
+// handleFiles returns true if we have handled the files, otherwise return false
+func (b *Birc) handleFiles(msg *config.Message) bool {
+	if msg.Extra == nil {
+		return false
+	}
+	for _, rmsg := range helper.HandleExtra(msg, b.General) {
+		b.Local <- rmsg
+	}
+	if len(msg.Extra["file"]) == 0 {
+		return false
+	}
+	for _, f := range msg.Extra["file"] {
+		fi := f.(config.FileInfo)
+		if fi.Comment != "" {
+			msg.Text += fi.Comment + " : "
+		}
+		if fi.URL != "" {
+			msg.Text = fi.URL
+			if fi.Comment != "" {
+				msg.Text = fi.Comment + " : " + fi.URL
+			}
+		}
+		b.Local <- config.Message{Text: msg.Text, Username: msg.Username, Channel: msg.Channel, Event: msg.Event}
+	}
+	return true
+}
+
+func (b *Birc) handleJoinPart(client *girc.Client, event girc.Event) {
+	if len(event.Params) == 0 {
+		b.Log.Debugf("handleJoinPart: empty Params? %#v", event)
+		return
+	}
+	channel := strings.ToLower(event.Params[0])
+	if event.Command == "KICK" && event.Params[1] == b.Nick {
+		b.Log.Infof("Got kicked from %s by %s", channel, event.Source.Name)
+		time.Sleep(time.Duration(b.GetInt("RejoinDelay")) * time.Second)
+		b.Remote <- config.Message{Username: "system", Text: "rejoin", Channel: channel, Account: b.Account, Event: config.EventRejoinChannels}
+		return
+	}
+	if event.Command == "QUIT" {
+		if event.Source.Name == b.Nick && strings.Contains(event.Last(), "Ping timeout") {
+			b.Log.Infof("%s reconnecting ..", b.Account)
+			b.Remote <- config.Message{Username: "system", Text: "reconnect", Channel: channel, Account: b.Account, Event: config.EventFailure}
+			return
+		}
+	}
+	if event.Source.Name != b.Nick {
+		if b.GetBool("nosendjoinpart") {
+			return
+		}
+		msg := config.Message{Username: "system", Text: event.Source.Name + " " + strings.ToLower(event.Command) + "s", Channel: channel, Account: b.Account, Event: config.EventJoinLeave}
+		if b.GetBool("verbosejoinpart") {
+			b.Log.Debugf("<= Sending verbose JOIN_LEAVE event from %s to gateway", b.Account)
+			msg = config.Message{Username: "system", Text: event.Source.Name + " (" + event.Source.Ident + "@" + event.Source.Host + ") " + strings.ToLower(event.Command) + "s", Channel: channel, Account: b.Account, Event: config.EventJoinLeave}
+		} else {
+			b.Log.Debugf("<= Sending JOIN_LEAVE event from %s to gateway", b.Account)
+		}
+		b.Log.Debugf("<= Message is %#v", msg)
+		b.Remote <- msg
+		return
+	}
+	b.Log.Debugf("handle %#v", event)
+}
+
+func (b *Birc) handleNewConnection(client *girc.Client, event girc.Event) {
+	b.Log.Debug("Registering callbacks")
+	i := b.i
+	b.Nick = event.Params[0]
+
+	i.Handlers.Add("PRIVMSG", b.handlePrivMsg)
+	i.Handlers.Add("CTCP_ACTION", b.handlePrivMsg)
+	i.Handlers.Add(girc.RPL_TOPICWHOTIME, b.handleTopicWhoTime)
+	i.Handlers.Add(girc.NOTICE, b.handleNotice)
+	i.Handlers.Add("JOIN", b.handleJoinPart)
+	i.Handlers.Add("PART", b.handleJoinPart)
+	i.Handlers.Add("QUIT", b.handleJoinPart)
+	i.Handlers.Add("KICK", b.handleJoinPart)
+}
+
+func (b *Birc) handleNickServ() {
+	if !b.GetBool("UseSASL") && b.GetString("NickServNick") != "" && b.GetString("NickServPassword") != "" {
+		b.Log.Debugf("Sending identify to nickserv %s", b.GetString("NickServNick"))
+		b.i.Cmd.Message(b.GetString("NickServNick"), "IDENTIFY "+b.GetString("NickServPassword"))
+	}
+	if strings.EqualFold(b.GetString("NickServNick"), "Q@CServe.quakenet.org") {
+		b.Log.Debugf("Authenticating %s against %s", b.GetString("NickServUsername"), b.GetString("NickServNick"))
+		b.i.Cmd.Message(b.GetString("NickServNick"), "AUTH "+b.GetString("NickServUsername")+" "+b.GetString("NickServPassword"))
+	}
+	// give nickserv some slack
+	time.Sleep(time.Second * 5)
+	b.authDone = true
+}
+
+func (b *Birc) handleNotice(client *girc.Client, event girc.Event) {
+	if strings.Contains(event.String(), "This nickname is registered") && event.Source.Name == b.GetString("NickServNick") {
+		b.handleNickServ()
+	} else {
+		b.handlePrivMsg(client, event)
+	}
+}
+
+func (b *Birc) handleOther(client *girc.Client, event girc.Event) {
+	if b.GetInt("DebugLevel") == 1 {
+		if event.Command != "CLIENT_STATE_UPDATED" &&
+			event.Command != "CLIENT_GENERAL_UPDATED" {
+			b.Log.Debugf("%#v", event.String())
+		}
+		return
+	}
+	switch event.Command {
+	case "372", "375", "376", "250", "251", "252", "253", "254", "255", "265", "266", "002", "003", "004", "005":
+		return
+	}
+	b.Log.Debugf("%#v", event.String())
+}
+
+func (b *Birc) handleOtherAuth(client *girc.Client, event girc.Event) {
+	b.handleNickServ()
+	b.handleRunCommands()
+	// we are now fully connected
+	// only send on first connection
+	if b.FirstConnection {
+		b.connected <- nil
+	}
+}
+
+func (b *Birc) handlePrivMsg(client *girc.Client, event girc.Event) {
+	if b.skipPrivMsg(event) {
+		return
+	}
+	rmsg := config.Message{Username: event.Source.Name, Channel: strings.ToLower(event.Params[0]), Account: b.Account, UserID: event.Source.Ident + "@" + event.Source.Host}
+	b.Log.Debugf("== Receiving PRIVMSG: %s %s %#v", event.Source.Name, event.Last(), event)
+
+	// set action event
+	if event.IsAction() {
+		rmsg.Event = config.EventUserAction
+	}
+
+	// strip action, we made an event if it was an action
+	rmsg.Text += event.StripAction()
+
+	// start detecting the charset
+	mycharset := b.GetString("Charset")
+	if mycharset == "" {
+		// detect what were sending so that we convert it to utf-8
+		detector := chardet.NewTextDetector()
+		result, err := detector.DetectBest([]byte(rmsg.Text))
+		if err != nil {
+			b.Log.Infof("detection failed for rmsg.Text: %#v", rmsg.Text)
+			return
+		}
+		b.Log.Debugf("detected %s confidence %#v", result.Charset, result.Confidence)
+		mycharset = result.Charset
+		// if we're not sure, just pick ISO-8859-1
+		if result.Confidence < 80 {
+			mycharset = "ISO-8859-1"
+		}
+	}
+	switch mycharset {
+	case "gbk", "gb18030", "gb2312", "big5", "euc-kr", "euc-jp", "shift-jis", "iso-2022-jp":
+		rmsg.Text = ic.ConvertString("utf-8", b.GetString("Charset"), rmsg.Text)
+	default:
+		r, err := charset.NewReader(mycharset, strings.NewReader(rmsg.Text))
+		if err != nil {
+			b.Log.Errorf("charset to utf-8 conversion failed: %s", err)
+			return
+		}
+		output, _ := ioutil.ReadAll(r)
+		rmsg.Text = string(output)
+	}
+
+	b.Log.Debugf("<= Sending message from %s on %s to gateway", event.Params[0], b.Account)
+	b.Remote <- rmsg
+}
+
+func (b *Birc) handleRunCommands() {
+	for _, cmd := range b.GetStringSlice("RunCommands") {
+		if err := b.i.Cmd.SendRaw(cmd); err != nil {
+			b.Log.Errorf("RunCommands %s failed: %s", cmd, err)
+		}
+		time.Sleep(time.Second)
+	}
+}
+
+func (b *Birc) handleTopicWhoTime(client *girc.Client, event girc.Event) {
+	parts := strings.Split(event.Params[2], "!")
+	t, err := strconv.ParseInt(event.Params[3], 10, 64)
+	if err != nil {
+		b.Log.Errorf("Invalid time stamp: %s", event.Params[3])
+	}
+	user := parts[0]
+	if len(parts) > 1 {
+		user += " [" + parts[1] + "]"
+	}
+	b.Log.Debugf("%s: Topic set by %s [%s]", event.Command, user, time.Unix(t, 0))
+}

bridge/irc/helper.go

@@ -1,61 +0,0 @@
-package birc
-
-import (
-	"strings"
-)
-
-/*
-func tableformatter(nicks []string, nicksPerRow int, continued bool) string {
-	result := "|IRC users"
-	if continued {
-		result = "|(continued)"
-	}
-	for i := 0; i < 2; i++ {
-		for j := 1; j <= nicksPerRow && j <= len(nicks); j++ {
-			if i == 0 {
-				result += "|"
-			} else {
-				result += ":-|"
-			}
-		}
-		result += "\r\n|"
-	}
-	result += nicks[0] + "|"
-	for i := 1; i < len(nicks); i++ {
-		if i%nicksPerRow == 0 {
-			result += "\r\n|" + nicks[i] + "|"
-		} else {
-			result += nicks[i] + "|"
-		}
-	}
-	return result
-}
-*/
-
-func plainformatter(nicks []string, nicksPerRow int) string {
-	return strings.Join(nicks, ", ") + " currently on IRC"
-}
-
-func IsMarkup(message string) bool {
-	switch message[0] {
-	case '|':
-		fallthrough
-	case '#':
-		fallthrough
-	case '_':
-		fallthrough
-	case '*':
-		fallthrough
-	case '~':
-		fallthrough
-	case '-':
-		fallthrough
-	case ':':
-		fallthrough
-	case '>':
-		fallthrough
-	case '=':
-		return true
-	}
-	return false
-}

bridge/irc/irc.go

@@ -1,37 +1,33 @@
 package birc
 
 import (
-	"bytes"
 	"crypto/tls"
 	"fmt"
 	"hash/crc32"
-	"io"
-	"io/ioutil"
 	"net"
-	"regexp"
 	"sort"
 	"strconv"
 	"strings"
 	"time"
-	"unicode/utf8"
 
 	"github.com/42wim/matterbridge/bridge"
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/42wim/matterbridge/bridge/helper"
-	"github.com/dfordsoft/golib/ic"
 	"github.com/lrstanley/girc"
-	"github.com/paulrosania/go-charset/charset"
+	stripmd "github.com/writeas/go-strip-markdown"
+
+	// We need to import the 'data' package as an implicit dependency.
+	// See: https://godoc.org/github.com/paulrosania/go-charset/charset
 	_ "github.com/paulrosania/go-charset/data"
-	"github.com/saintfish/chardet"
 )
 
 type Birc struct {
 	i                                         *girc.Client
 	Nick                                      string
 	names                                     map[string][]string
-	connected                                 chan struct{}
+	connected                                 chan error
 	Local                                     chan config.Message // local queue for flood control
-	FirstConnection                           bool
+	FirstConnection, authDone                 bool
 	MessageDelay, MessageQueue, MessageLength int
 
 	*bridge.Config
@@ -42,7 +38,7 @@ func New(cfg *bridge.Config) bridge.Bridger {
 	b.Config = cfg
 	b.Nick = b.GetString("Nick")
 	b.names = make(map[string][]string)
-	b.connected = make(chan struct{})
+	b.connected = make(chan error)
 	if b.GetInt("MessageDelay") == 0 {
 		b.MessageDelay = 1300
 	} else {
@@ -63,11 +59,10 @@ func New(cfg *bridge.Config) bridge.Bridger {
 }
 
 func (b *Birc) Command(msg *config.Message) string {
-	switch msg.Text {
+	if msg.Text == "!users" {
-	case "!users":
 		b.i.Handlers.Add(girc.RPL_NAMREPLY, b.storeNames)
 		b.i.Handlers.Add(girc.RPL_ENDOFNAMES, b.endNames)
-		b.i.Cmd.SendRaw("NAMES " + msg.Channel)
+		b.i.Cmd.SendRaw("NAMES " + msg.Channel) //nolint:errcheck
 	}
 	return ""
 }
@@ -75,69 +70,33 @@ func (b *Birc) Command(msg *config.Message) string {
 func (b *Birc) Connect() error {
 	b.Local = make(chan config.Message, b.MessageQueue+10)
 	b.Log.Infof("Connecting %s", b.GetString("Server"))
-	server, portstr, err := net.SplitHostPort(b.GetString("Server"))
-	if err != nil {
-		return err
-	}
-	port, err := strconv.Atoi(portstr)
-	if err != nil {
-		return err
-	}
-	// fix strict user handling of girc
-	user := b.GetString("Nick")
-	for !girc.IsValidUser(user) {
-		if len(user) == 1 {
-			user = "matterbridge"
-			break
-		}
-		user = user[1:]
-	}
 
-	i := girc.New(girc.Config{
+	i, err := b.getClient()
-		Server:     server,
+	if err != nil {
-		ServerPass: b.GetString("Password"),
+		return err
-		Port:       port,
+	}
-		Nick:       b.GetString("Nick"),
-		User:       user,
-		Name:       b.GetString("Nick"),
-		SSL:        b.GetBool("UseTLS"),
-		TLSConfig:  &tls.Config{InsecureSkipVerify: b.GetBool("SkipTLSVerify"), ServerName: server},
-		PingDelay:  time.Minute,
-	})
 
 	if b.GetBool("UseSASL") {
-		i.Config.SASL = &girc.SASLPlain{b.GetString("NickServNick"), b.GetString("NickServPassword")}
+		i.Config.SASL = &girc.SASLPlain{
+			User: b.GetString("NickServNick"),
+			Pass: b.GetString("NickServPassword"),
+		}
 	}
 
 	i.Handlers.Add(girc.RPL_WELCOME, b.handleNewConnection)
 	i.Handlers.Add(girc.RPL_ENDOFMOTD, b.handleOtherAuth)
+	i.Handlers.Add(girc.ERR_NOMOTD, b.handleOtherAuth)
 	i.Handlers.Add(girc.ALL_EVENTS, b.handleOther)
-	go func() {
-		for {
-			if err := i.Connect(); err != nil {
-				b.Log.Errorf("disconnect: error: %s", err)
-			} else {
-				b.Log.Info("disconnect: client requested quit")
-			}
-
-			b.Log.Info("reconnecting in 30 seconds...")
-			time.Sleep(30 * time.Second)
-			i.Handlers.Clear(girc.RPL_WELCOME)
-			i.Handlers.Add(girc.RPL_WELCOME, func(client *girc.Client, event girc.Event) {
-				b.Remote <- config.Message{Username: "system", Text: "rejoin", Channel: "", Account: b.Account, Event: config.EVENT_REJOIN_CHANNELS}
-				// set our correct nick on reconnect if necessary
-				b.Nick = event.Source.Name
-			})
-		}
-	}()
 	b.i = i
-	select {
+
-	case <-b.connected:
+	go b.doConnect()
-		b.Log.Info("Connection succeeded")
+
-	case <-time.After(time.Second * 30):
+	err = <-b.connected
-		return fmt.Errorf("connection timed out")
+	if err != nil {
+		return fmt.Errorf("connection failed %s", err)
 	}
-	//i.Debug = false
+	b.Log.Info("Connection succeeded")
+	b.FirstConnection = false
 	if b.GetInt("DebugLevel") == 0 {
 		i.Handlers.Clear(girc.ALL_EVENTS)
 	}
@@ -152,6 +111,13 @@ func (b *Birc) Disconnect() error {
 }
 
 func (b *Birc) JoinChannel(channel config.ChannelInfo) error {
+	// need to check if we have nickserv auth done before joining channels
+	for {
+		if b.authDone {
+			break
+		}
+		time.Sleep(time.Second)
+	}
 	if channel.Options.Key != "" {
 		b.Log.Debugf("using key %s for channel %s", channel.Options.Key, channel.Name)
 		b.i.Cmd.JoinKey(channel.Name, channel.Options.Key)
@@ -163,7 +129,7 @@ func (b *Birc) JoinChannel(channel config.ChannelInfo) error {
 
 func (b *Birc) Send(msg config.Message) (string, error) {
 	// ignore delete messages
-	if msg.Event == config.EVENT_MSG_DELETE {
+	if msg.Event == config.EventMsgDelete {
 		return "", nil
 	}
 
@@ -172,6 +138,7 @@ func (b *Birc) Send(msg config.Message) (string, error) {
 	// we can be in between reconnects #385
 	if !b.i.IsConnected() {
 		b.Log.Error("Not connected to server, dropping message")
+		return "", nil
 	}
 
 	// Execute a command
@@ -180,82 +147,71 @@ func (b *Birc) Send(msg config.Message) (string, error) {
 	}
 
 	// convert to specified charset
-	if b.GetString("Charset") != "" {
+	if err := b.handleCharset(&msg); err != nil {
-		switch b.GetString("Charset") {
+		return "", err
-		case "gbk", "gb18030", "gb2312", "big5", "euc-kr", "euc-jp", "shift-jis", "iso-2022-jp":
-			msg.Text = ic.ConvertString("utf-8", b.GetString("Charset"), msg.Text)
-		default:
-			buf := new(bytes.Buffer)
-			w, err := charset.NewWriter(b.GetString("Charset"), buf)
-			if err != nil {
-				b.Log.Errorf("charset from utf-8 conversion failed: %s", err)
-				return "", err
-			}
-			fmt.Fprint(w, msg.Text)
-			w.Close()
-			msg.Text = buf.String()
-		}
 	}
 
-	// Handle files
+	// handle files, return if we're done here
-	if msg.Extra != nil {
+	if ok := b.handleFiles(&msg); ok {
-		for _, rmsg := range helper.HandleExtra(&msg, b.General) {
+		return "", nil
-			b.Local <- rmsg
+	}
-		}
+
-		if len(msg.Extra["file"]) > 0 {
+	var msgLines []string
-			for _, f := range msg.Extra["file"] {
+	if b.GetBool("StripMarkdown") {
-				fi := f.(config.FileInfo)
+		msg.Text = stripmd.Strip(msg.Text)
-				if fi.Comment != "" {
+	}
-					msg.Text += fi.Comment + ": "
+
-				}
+	if b.GetBool("MessageSplit") {
-				if fi.URL != "" {
+		msgLines = helper.GetSubLines(msg.Text, b.MessageLength)
-					msg.Text = fi.URL
+	} else {
-					if fi.Comment != "" {
+		msgLines = helper.GetSubLines(msg.Text, 0)
-						msg.Text = fi.Comment + ": " + fi.URL
+	}
-					}
+	for i := range msgLines {
-				}
+		if len(b.Local) >= b.MessageQueue {
-				b.Local <- config.Message{Text: msg.Text, Username: msg.Username, Channel: msg.Channel, Event: msg.Event}
+			b.Log.Debugf("flooding, dropping message (queue at %d)", len(b.Local))
-			}
 			return "", nil
 		}
-	}
 
-	// split long messages on messageLength, to avoid clipped messages #281
+		msg.Text = msgLines[i]
-	if b.GetBool("MessageSplit") {
+		b.Local <- msg
-		msg.Text = helper.SplitStringLength(msg.Text, b.MessageLength)
-	}
-	for _, text := range strings.Split(msg.Text, "\n") {
-		if len(text) > b.MessageLength {
-			text = text[:b.MessageLength-len(" <message clipped>")]
-			if r, size := utf8.DecodeLastRuneInString(text); r == utf8.RuneError {
-				text = text[:len(text)-size]
-			}
-			text += " <message clipped>"
-		}
-		if len(b.Local) < b.MessageQueue {
-			if len(b.Local) == b.MessageQueue-1 {
-				text = text + " <message clipped>"
-			}
-			b.Local <- config.Message{Text: text, Username: msg.Username, Channel: msg.Channel, Event: msg.Event}
-		} else {
-			b.Log.Debugf("flooding, dropping message (queue at %d)", len(b.Local))
-		}
 	}
 	return "", nil
 }
 
+func (b *Birc) doConnect() {
+	for {
+		if err := b.i.Connect(); err != nil {
+			b.Log.Errorf("disconnect: error: %s", err)
+			if b.FirstConnection {
+				b.connected <- err
+				return
+			}
+		} else {
+			b.Log.Info("disconnect: client requested quit")
+		}
+		b.Log.Info("reconnecting in 30 seconds...")
+		time.Sleep(30 * time.Second)
+		b.i.Handlers.Clear(girc.RPL_WELCOME)
+		b.i.Handlers.Add(girc.RPL_WELCOME, func(client *girc.Client, event girc.Event) {
+			b.Remote <- config.Message{Username: "system", Text: "rejoin", Channel: "", Account: b.Account, Event: config.EventRejoinChannels}
+			// set our correct nick on reconnect if necessary
+			b.Nick = event.Source.Name
+		})
+	}
+}
+
 func (b *Birc) doSend() {
 	rate := time.Millisecond * time.Duration(b.MessageDelay)
 	throttle := time.NewTicker(rate)
 	for msg := range b.Local {
 		<-throttle.C
 		username := msg.Username
-		if b.GetBool("Colornicks") {
+		if b.GetBool("Colornicks") && len(username) > 1 {
 			checksum := crc32.ChecksumIEEE([]byte(msg.Username))
 			colorCode := checksum%14 + 2 // quick fix - prevent white or black color codes
 			username = fmt.Sprintf("\x03%02d%s\x0F", colorCode, msg.Username)
 		}
-		if msg.Event == config.EVENT_USER_ACTION {
+		if msg.Event == config.EventUserAction {
 			b.i.Cmd.Action(msg.Channel, username+msg.Text)
 		} else {
 			b.Log.Debugf("Sending to channel %s", msg.Channel)
@@ -264,104 +220,58 @@ func (b *Birc) doSend() {
 	}
 }
 
+// validateInput validates the server/port/nick configuration. Returns a *girc.Client if successful
+func (b *Birc) getClient() (*girc.Client, error) {
+	server, portstr, err := net.SplitHostPort(b.GetString("Server"))
+	if err != nil {
+		return nil, err
+	}
+	port, err := strconv.Atoi(portstr)
+	if err != nil {
+		return nil, err
+	}
+	// fix strict user handling of girc
+	user := b.GetString("Nick")
+	for !girc.IsValidUser(user) {
+		if len(user) == 1 || len(user) == 0 {
+			user = "matterbridge"
+			break
+		}
+		user = user[1:]
+	}
+
+	i := girc.New(girc.Config{
+		Server:     server,
+		ServerPass: b.GetString("Password"),
+		Port:       port,
+		Nick:       b.GetString("Nick"),
+		User:       user,
+		Name:       b.GetString("Nick"),
+		SSL:        b.GetBool("UseTLS"),
+		TLSConfig:  &tls.Config{InsecureSkipVerify: b.GetBool("SkipTLSVerify"), ServerName: server}, //nolint:gosec
+		PingDelay:  time.Minute,
+		// skip gIRC internal rate limiting, since we have our own throttling
+		AllowFlood: true,
+	})
+	return i, nil
+}
+
 func (b *Birc) endNames(client *girc.Client, event girc.Event) {
 	channel := event.Params[1]
 	sort.Strings(b.names[channel])
 	maxNamesPerPost := (300 / b.nicksPerRow()) * b.nicksPerRow()
-	continued := false
 	for len(b.names[channel]) > maxNamesPerPost {
-		b.Remote <- config.Message{Username: b.Nick, Text: b.formatnicks(b.names[channel][0:maxNamesPerPost], continued),
+		b.Remote <- config.Message{Username: b.Nick, Text: b.formatnicks(b.names[channel][0:maxNamesPerPost]),
 			Channel: channel, Account: b.Account}
 		b.names[channel] = b.names[channel][maxNamesPerPost:]
-		continued = true
 	}
-	b.Remote <- config.Message{Username: b.Nick, Text: b.formatnicks(b.names[channel], continued),
+	b.Remote <- config.Message{Username: b.Nick, Text: b.formatnicks(b.names[channel]),
 		Channel: channel, Account: b.Account}
 	b.names[channel] = nil
 	b.i.Handlers.Clear(girc.RPL_NAMREPLY)
 	b.i.Handlers.Clear(girc.RPL_ENDOFNAMES)
 }
 
-func (b *Birc) handleNewConnection(client *girc.Client, event girc.Event) {
-	b.Log.Debug("Registering callbacks")
-	i := b.i
-	b.Nick = event.Params[0]
-
-	i.Handlers.Add(girc.RPL_ENDOFMOTD, b.handleOtherAuth)
-	i.Handlers.Add("PRIVMSG", b.handlePrivMsg)
-	i.Handlers.Add("CTCP_ACTION", b.handlePrivMsg)
-	i.Handlers.Add(girc.RPL_TOPICWHOTIME, b.handleTopicWhoTime)
-	i.Handlers.Add(girc.NOTICE, b.handleNotice)
-	i.Handlers.Add("JOIN", b.handleJoinPart)
-	i.Handlers.Add("PART", b.handleJoinPart)
-	i.Handlers.Add("QUIT", b.handleJoinPart)
-	i.Handlers.Add("KICK", b.handleJoinPart)
-	// we are now fully connected
-	b.connected <- struct{}{}
-}
-
-func (b *Birc) handleJoinPart(client *girc.Client, event girc.Event) {
-	if len(event.Params) == 0 {
-		b.Log.Debugf("handleJoinPart: empty Params? %#v", event)
-		return
-	}
-	channel := strings.ToLower(event.Params[0])
-	if event.Command == "KICK" {
-		b.Log.Infof("Got kicked from %s by %s", channel, event.Source.Name)
-		time.Sleep(time.Duration(b.GetInt("RejoinDelay")) * time.Second)
-		b.Remote <- config.Message{Username: "system", Text: "rejoin", Channel: channel, Account: b.Account, Event: config.EVENT_REJOIN_CHANNELS}
-		return
-	}
-	if event.Command == "QUIT" {
-		if event.Source.Name == b.Nick && strings.Contains(event.Trailing, "Ping timeout") {
-			b.Log.Infof("%s reconnecting ..", b.Account)
-			b.Remote <- config.Message{Username: "system", Text: "reconnect", Channel: channel, Account: b.Account, Event: config.EVENT_FAILURE}
-			return
-		}
-	}
-	if event.Source.Name != b.Nick {
-		if b.GetBool("nosendjoinpart") {
-			return
-		}
-		b.Log.Debugf("<= Sending JOIN_LEAVE event from %s to gateway", b.Account)
-		msg := config.Message{Username: "system", Text: event.Source.Name + " " + strings.ToLower(event.Command) + "s", Channel: channel, Account: b.Account, Event: config.EVENT_JOIN_LEAVE}
-		b.Log.Debugf("<= Message is %#v", msg)
-		b.Remote <- msg
-		return
-	}
-	b.Log.Debugf("handle %#v", event)
-}
-
-func (b *Birc) handleNotice(client *girc.Client, event girc.Event) {
-	if strings.Contains(event.String(), "This nickname is registered") && event.Source.Name == b.GetString("NickServNick") {
-		b.i.Cmd.Message(b.GetString("NickServNick"), "IDENTIFY "+b.GetString("NickServPassword"))
-	} else {
-		b.handlePrivMsg(client, event)
-	}
-}
-
-func (b *Birc) handleOther(client *girc.Client, event girc.Event) {
-	if b.GetInt("DebugLevel") == 1 {
-		if event.Command != "CLIENT_STATE_UPDATED" &&
-			event.Command != "CLIENT_GENERAL_UPDATED" {
-			b.Log.Debugf("%#v", event.String())
-		}
-		return
-	}
-	switch event.Command {
-	case "372", "375", "376", "250", "251", "252", "253", "254", "255", "265", "266", "002", "003", "004", "005":
-		return
-	}
-	b.Log.Debugf("%#v", event.String())
-}
-
-func (b *Birc) handleOtherAuth(client *girc.Client, event girc.Event) {
-	if strings.EqualFold(b.GetString("NickServNick"), "Q@CServe.quakenet.org") {
-		b.Log.Debugf("Authenticating %s against %s", b.GetString("NickServUsername"), b.GetString("NickServNick"))
-		b.i.Cmd.Message(b.GetString("NickServNick"), "AUTH "+b.GetString("NickServUsername")+" "+b.GetString("NickServPassword"))
-	}
-}
-
 func (b *Birc) skipPrivMsg(event girc.Event) bool {
 	// Our nick can be changed
 	b.Nick = b.i.GetNick()
@@ -381,74 +291,6 @@ func (b *Birc) skipPrivMsg(event girc.Event) bool {
 	return false
 }
 
-func (b *Birc) handlePrivMsg(client *girc.Client, event girc.Event) {
-	if b.skipPrivMsg(event) {
-		return
-	}
-	rmsg := config.Message{Username: event.Source.Name, Channel: strings.ToLower(event.Params[0]), Account: b.Account, UserID: event.Source.Ident + "@" + event.Source.Host}
-	b.Log.Debugf("== Receiving PRIVMSG: %s %s %#v", event.Source.Name, event.Trailing, event)
-
-	// set action event
-	if event.IsAction() {
-		rmsg.Event = config.EVENT_USER_ACTION
-	}
-
-	// strip action, we made an event if it was an action
-	rmsg.Text += event.StripAction()
-
-	// strip IRC colors
-	re := regexp.MustCompile(`[[:cntrl:]](?:\d{1,2}(?:,\d{1,2})?)?`)
-	rmsg.Text = re.ReplaceAllString(rmsg.Text, "")
-
-	// start detecting the charset
-	var r io.Reader
-	var err error
-	mycharset := b.GetString("Charset")
-	if mycharset == "" {
-		// detect what were sending so that we convert it to utf-8
-		detector := chardet.NewTextDetector()
-		result, err := detector.DetectBest([]byte(rmsg.Text))
-		if err != nil {
-			b.Log.Infof("detection failed for rmsg.Text: %#v", rmsg.Text)
-			return
-		}
-		b.Log.Debugf("detected %s confidence %#v", result.Charset, result.Confidence)
-		mycharset = result.Charset
-		// if we're not sure, just pick ISO-8859-1
-		if result.Confidence < 80 {
-			mycharset = "ISO-8859-1"
-		}
-	}
-	switch mycharset {
-	case "gbk", "gb18030", "gb2312", "big5", "euc-kr", "euc-jp", "shift-jis", "iso-2022-jp":
-		rmsg.Text = ic.ConvertString("utf-8", b.GetString("Charset"), rmsg.Text)
-	default:
-		r, err = charset.NewReader(mycharset, strings.NewReader(rmsg.Text))
-		if err != nil {
-			b.Log.Errorf("charset to utf-8 conversion failed: %s", err)
-			return
-		}
-		output, _ := ioutil.ReadAll(r)
-		rmsg.Text = string(output)
-	}
-
-	b.Log.Debugf("<= Sending message from %s on %s to gateway", event.Params[0], b.Account)
-	b.Remote <- rmsg
-}
-
-func (b *Birc) handleTopicWhoTime(client *girc.Client, event girc.Event) {
-	parts := strings.Split(event.Params[2], "!")
-	t, err := strconv.ParseInt(event.Params[3], 10, 64)
-	if err != nil {
-		b.Log.Errorf("Invalid time stamp: %s", event.Params[3])
-	}
-	user := parts[0]
-	if len(parts) > 1 {
-		user += " [" + parts[1] + "]"
-	}
-	b.Log.Debugf("%s: Topic set by %s [%s]", event.Command, user, time.Unix(t, 0))
-}
-
 func (b *Birc) nicksPerRow() int {
 	return 4
 }
@@ -457,9 +299,9 @@ func (b *Birc) storeNames(client *girc.Client, event girc.Event) {
 	channel := event.Params[2]
 	b.names[channel] = append(
 		b.names[channel],
-		strings.Split(strings.TrimSpace(event.Trailing), " ")...)
+		strings.Split(strings.TrimSpace(event.Last()), " ")...)
 }
 
-func (b *Birc) formatnicks(nicks []string, continued bool) string {
+func (b *Birc) formatnicks(nicks []string) string {
-	return plainformatter(nicks, b.nicksPerRow())
+	return strings.Join(nicks, ", ") + " currently on IRC"
 }

bridge/keybase/handlers.go

@@ -0,0 +1,59 @@
+package bkeybase
+
+import (
+	"strconv"
+
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/keybase/go-keybase-chat-bot/kbchat/types/chat1"
+)
+
+func (b *Bkeybase) handleKeybase() {
+	sub, err := b.kbc.ListenForNewTextMessages()
+	if err != nil {
+		b.Log.Errorf("Error listening: %s", err.Error())
+	}
+
+	go func() {
+		for {
+			msg, err := sub.Read()
+			if err != nil {
+				b.Log.Errorf("failed to read message: %s", err.Error())
+			}
+
+			if msg.Message.Content.TypeName != "text" {
+				continue
+			}
+
+			if msg.Message.Sender.Username == b.kbc.GetUsername() {
+				continue
+			}
+
+			b.handleMessage(msg.Message)
+
+		}
+	}()
+}
+
+func (b *Bkeybase) handleMessage(msg chat1.MsgSummary) {
+	b.Log.Debugf("== Receiving event: %#v", msg)
+	if msg.Channel.TopicName != b.channel || msg.Channel.Name != b.team {
+		return
+	}
+
+	if msg.Sender.Username != b.kbc.GetUsername() {
+
+		// TODO download avatar
+
+		// Create our message
+		rmsg := config.Message{Username: msg.Sender.Username, Text: msg.Content.Text.Body, UserID: string(msg.Sender.Uid), Channel: msg.Channel.TopicName, ID: strconv.Itoa(int(msg.Id)), Account: b.Account}
+
+		// Text must be a string
+		if msg.Content.TypeName != "text" {
+			b.Log.Errorf("message is not text")
+			return
+		}
+
+		b.Log.Debugf("<= Sending message from %s on %s to gateway", msg.Sender.Username, msg.Channel.Name)
+		b.Remote <- rmsg
+	}
+}

bridge/keybase/keybase.go

@@ -0,0 +1,106 @@
+package bkeybase
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+
+	"github.com/42wim/matterbridge/bridge"
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/keybase/go-keybase-chat-bot/kbchat"
+)
+
+// Bkeybase bridge structure
+type Bkeybase struct {
+	kbc     *kbchat.API
+	user    string
+	channel string
+	team    string
+	*bridge.Config
+}
+
+// New initializes Bkeybase object and sets team
+func New(cfg *bridge.Config) bridge.Bridger {
+	b := &Bkeybase{Config: cfg}
+	b.team = b.Config.GetString("Team")
+	return b
+}
+
+// Connect starts keybase API and listener loop
+func (b *Bkeybase) Connect() error {
+	var err error
+	b.Log.Infof("Connecting %s", b.GetString("Team"))
+
+	// use default keybase location (`keybase`)
+	b.kbc, err = kbchat.Start(kbchat.RunOptions{})
+	if err != nil {
+		return err
+	}
+	b.user = b.kbc.GetUsername()
+	b.Log.Info("Connection succeeded")
+	go b.handleKeybase()
+	return nil
+}
+
+// Disconnect doesn't do anything for now
+func (b *Bkeybase) Disconnect() error {
+	return nil
+}
+
+// JoinChannel sets channel name in struct
+func (b *Bkeybase) JoinChannel(channel config.ChannelInfo) error {
+	if _, err := b.kbc.JoinChannel(b.team, channel.Name); err != nil {
+		return err
+	}
+	b.channel = channel.Name
+	return nil
+}
+
+// Send receives bridge messages and sends them to Keybase chat room
+func (b *Bkeybase) Send(msg config.Message) (string, error) {
+	b.Log.Debugf("=> Receiving %#v", msg)
+
+	// Handle /me events
+	if msg.Event == config.EventUserAction {
+		msg.Text = "_" + msg.Text + "_"
+	}
+
+	// Delete message if we have an ID
+	// Delete message not supported by keybase go library yet
+
+	// Edit message if we have an ID
+	// kbchat lib does not support message editing yet
+
+	if len(msg.Extra["file"]) > 0 {
+		// Upload a file
+		dir, err := ioutil.TempDir("", "matterbridge")
+		if err != nil {
+			return "", err
+		}
+		defer os.RemoveAll(dir)
+
+		for _, f := range msg.Extra["file"] {
+			fname := f.(config.FileInfo).Name
+			fdata := *f.(config.FileInfo).Data
+			fcaption := f.(config.FileInfo).Comment
+			fpath := filepath.Join(dir, fname)
+
+			if err = ioutil.WriteFile(fpath, fdata, 0600); err != nil {
+				return "", err
+			}
+
+			_, _ = b.kbc.SendAttachmentByTeam(b.team, &b.channel, fpath, fcaption)
+		}
+
+		return "", nil
+	}
+
+	// Send regular message
+	text := msg.Username + msg.Text
+	resp, err := b.kbc.SendMessageByTeamName(b.team, &b.channel, text)
+	if err != nil {
+		return "", err
+	}
+	return strconv.Itoa(int(*resp.Result.MessageID)), err
+}

bridge/matrix/matrix.go

@@ -2,16 +2,19 @@ package bmatrix
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
+	"html"
 	"mime"
 	"regexp"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/42wim/matterbridge/bridge"
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/42wim/matterbridge/bridge/helper"
-	matrix "github.com/matterbridge/gomatrix"
+	matrix "github.com/matrix-org/gomatrix"
 )
 
 type Bmatrix struct {
@@ -19,11 +22,21 @@ type Bmatrix struct {
 	UserID  string
 	RoomMap map[string]string
 	sync.RWMutex
+	htmlTag            *regexp.Regexp
+	htmlReplacementTag *regexp.Regexp
 	*bridge.Config
 }
 
+type httpError struct {
+	Errcode      string `json:"errcode"`
+	Err          string `json:"error"`
+	RetryAfterMs int    `json:"retry_after_ms"`
+}
+
 func New(cfg *bridge.Config) bridge.Bridger {
 	b := &Bmatrix{Config: cfg}
+	b.htmlTag = regexp.MustCompile("</.*?>")
+	b.htmlReplacementTag = regexp.MustCompile("<[^>]*>")
 	b.RoomMap = make(map[string]string)
 	return b
 }
@@ -36,9 +49,10 @@ func (b *Bmatrix) Connect() error {
 		return err
 	}
 	resp, err := b.mc.Login(&matrix.ReqLogin{
-		Type:     "m.login.password",
+		Type:       "m.login.password",
-		User:     b.GetString("Login"),
+		User:       b.GetString("Login"),
-		Password: b.GetString("Password"),
+		Password:   b.GetString("Password"),
+		Identifier: matrix.NewUserIdentifier(b.GetString("Login")),
 	})
 	if err != nil {
 		return err
@@ -55,14 +69,25 @@ func (b *Bmatrix) Disconnect() error {
 }
 
 func (b *Bmatrix) JoinChannel(channel config.ChannelInfo) error {
+retry:
 	resp, err := b.mc.JoinRoom(channel.Name, "", nil)
 	if err != nil {
+		httpErr := handleError(err)
+		if httpErr.Errcode == "M_LIMIT_EXCEEDED" {
+			b.Log.Infof("getting ratelimited by matrix, sleeping approx %d seconds before joining %s", httpErr.RetryAfterMs/1000, channel.Name)
+			time.Sleep((time.Duration(httpErr.RetryAfterMs) * time.Millisecond))
+
+			goto retry
+		}
+
 		return err
 	}
+
 	b.Lock()
 	b.RoomMap[resp.RoomID] = channel.Name
 	b.Unlock()
-	return err
+
+	return nil
 }
 
 func (b *Bmatrix) Send(msg config.Message) (string, error) {
@@ -72,9 +97,12 @@ func (b *Bmatrix) Send(msg config.Message) (string, error) {
 	b.Log.Debugf("Channel %s maps to channel id %s", msg.Channel, channel)
 
 	// Make a action /me of the message
-	if msg.Event == config.EVENT_USER_ACTION {
+	if msg.Event == config.EventUserAction {
-		resp, err := b.mc.SendMessageEvent(channel, "m.room.message",
+		m := matrix.TextMessage{
-			matrix.TextMessage{"m.emote", msg.Username + msg.Text})
+			MsgType: "m.emote",
+			Body:    msg.Username + msg.Text,
+		}
+		resp, err := b.mc.SendMessageEvent(channel, "m.room.message", m)
 		if err != nil {
 			return "", err
 		}
@@ -82,7 +110,7 @@ func (b *Bmatrix) Send(msg config.Message) (string, error) {
 	}
 
 	// Delete message
-	if msg.Event == config.EVENT_MSG_DELETE {
+	if msg.Event == config.EventMsgDelete {
 		if msg.ID == "" {
 			return "", nil
 		}
@@ -96,19 +124,50 @@ func (b *Bmatrix) Send(msg config.Message) (string, error) {
 	// Upload a file if it exists
 	if msg.Extra != nil {
 		for _, rmsg := range helper.HandleExtra(&msg, b.General) {
-			b.mc.SendText(channel, rmsg.Username+rmsg.Text)
+			if _, err := b.mc.SendText(channel, rmsg.Username+rmsg.Text); err != nil {
+				b.Log.Errorf("sendText failed: %s", err)
+			}
 		}
 		// check if we have files to upload (from slack, telegram or mattermost)
 		if len(msg.Extra["file"]) > 0 {
-			return b.handleUploadFile(&msg, channel)
+			return b.handleUploadFiles(&msg, channel)
 		}
 	}
 
 	// Edit message if we have an ID
 	// matrix has no editing support
 
-	// Post normal message
+	// Use notices to send join/leave events
-	resp, err := b.mc.SendText(channel, msg.Username+msg.Text)
+	if msg.Event == config.EventJoinLeave {
+		resp, err := b.mc.SendNotice(channel, msg.Username+msg.Text)
+		if err != nil {
+			return "", err
+		}
+		return resp.EventID, err
+	}
+
+	if b.GetBool("HTMLDisable") {
+		resp, err := b.mc.SendText(channel, msg.Username+msg.Text)
+		if err != nil {
+			return "", err
+		}
+		return resp.EventID, err
+	}
+
+	var username string
+	var plainUsername string
+	// check if we have a </tag>. if we have, we don't escape HTML. #696
+	if b.htmlTag.MatchString(msg.Username) {
+		username = msg.Username
+		// remove the HTML formatting for beautiful push messages #1188
+		plainUsername = b.htmlReplacementTag.ReplaceAllString(msg.Username, "")
+	} else {
+		username = html.EscapeString(msg.Username)
+		plainUsername = msg.Username
+	}
+
+	// Post normal message with HTML support (eg riot.im)
+	resp, err := b.mc.SendFormattedText(channel, plainUsername+msg.Text, username+helper.ParseMarkdown(msg.Text))
 	if err != nil {
 		return "", err
 	}
@@ -126,7 +185,7 @@ func (b *Bmatrix) getRoomID(channel string) string {
 	return ""
 }
 
-func (b *Bmatrix) handlematrix() error {
+func (b *Bmatrix) handlematrix() {
 	syncer := b.mc.Syncer.(*matrix.DefaultSyncer)
 	syncer.OnEventType("m.room.redaction", b.handleEvent)
 	syncer.OnEventType("m.room.message", b.handleEvent)
@@ -137,7 +196,6 @@ func (b *Bmatrix) handlematrix() error {
 			}
 		}
 	}()
-	return nil
 }
 
 func (b *Bmatrix) handleEvent(ev *matrix.Event) {
@@ -151,14 +209,20 @@ func (b *Bmatrix) handleEvent(ev *matrix.Event) {
 			return
 		}
 
-		// TODO download avatar
-
 		// Create our message
-		rmsg := config.Message{Username: ev.Sender[1:], Channel: channel, Account: b.Account, UserID: ev.Sender, ID: ev.ID}
+		rmsg := config.Message{
+			Username: ev.Sender[1:],
+			Channel:  channel,
+			Account:  b.Account,
+			UserID:   ev.Sender,
+			ID:       ev.ID,
+			Avatar:   b.getAvatarURL(ev.Sender),
+		}
 
 		// Text must be a string
 		if rmsg.Text, ok = ev.Content["body"].(string); !ok {
-			b.Log.Errorf("Content[body] wasn't a %T ?", rmsg.Text)
+			b.Log.Errorf("Content[body] is not a string: %T\n%#v",
+				ev.Content["body"], ev.Content)
 			return
 		}
 
@@ -170,16 +234,16 @@ func (b *Bmatrix) handleEvent(ev *matrix.Event) {
 
 		// Delete event
 		if ev.Type == "m.room.redaction" {
-			rmsg.Event = config.EVENT_MSG_DELETE
+			rmsg.Event = config.EventMsgDelete
 			rmsg.ID = ev.Redacts
-			rmsg.Text = config.EVENT_MSG_DELETE
+			rmsg.Text = config.EventMsgDelete
 			b.Remote <- rmsg
 			return
 		}
 
 		// Do we have a /me action
 		if ev.Content["msgtype"].(string) == "m.emote" {
-			rmsg.Event = config.EVENT_USER_ACTION
+			rmsg.Event = config.EventUserAction
 		}
 
 		// Do we have attachments
@@ -231,11 +295,11 @@ func (b *Bmatrix) handleDownloadFile(rmsg *config.Message, content map[string]in
 		if msgtype == "m.image" {
 			mext, _ := mime.ExtensionsByType(mtype)
 			if len(mext) > 0 {
-				name = name + mext[0]
+				name += mext[0]
 			}
 		} else {
 			// just a default .png extension if we don't have mime info
-			name = name + ".png"
+			name += ".png"
 		}
 	}
 
@@ -254,47 +318,89 @@ func (b *Bmatrix) handleDownloadFile(rmsg *config.Message, content map[string]in
 	return nil
 }
 
-// handleUploadFile handles native upload of files
+// handleUploadFiles handles native upload of files.
-func (b *Bmatrix) handleUploadFile(msg *config.Message, channel string) (string, error) {
+func (b *Bmatrix) handleUploadFiles(msg *config.Message, channel string) (string, error) {
 	for _, f := range msg.Extra["file"] {
-		fi := f.(config.FileInfo)
+		if fi, ok := f.(config.FileInfo); ok {
-		content := bytes.NewReader(*fi.Data)
+			b.handleUploadFile(msg, channel, &fi)
-		sp := strings.Split(fi.Name, ".")
-		mtype := mime.TypeByExtension("." + sp[len(sp)-1])
-		if strings.Contains(mtype, "image") ||
-			strings.Contains(mtype, "video") {
-			if fi.Comment != "" {
-				_, err := b.mc.SendText(channel, msg.Username+fi.Comment)
-				if err != nil {
-					b.Log.Errorf("file comment failed: %#v", err)
-				}
-			}
-			b.Log.Debugf("uploading file: %s %s", fi.Name, mtype)
-			res, err := b.mc.UploadToContentRepo(content, mtype, int64(len(*fi.Data)))
-			if err != nil {
-				b.Log.Errorf("file upload failed: %#v", err)
-				continue
-			}
-			if strings.Contains(mtype, "video") {
-				b.Log.Debugf("sendVideo %s", res.ContentURI)
-				_, err = b.mc.SendVideo(channel, fi.Name, res.ContentURI)
-				if err != nil {
-					b.Log.Errorf("sendVideo failed: %#v", err)
-				}
-			}
-			if strings.Contains(mtype, "image") {
-				b.Log.Debugf("sendImage %s", res.ContentURI)
-				_, err = b.mc.SendImage(channel, fi.Name, res.ContentURI)
-				if err != nil {
-					b.Log.Errorf("sendImage failed: %#v", err)
-				}
-			}
-			b.Log.Debugf("result: %#v", res)
 		}
 	}
 	return "", nil
 }
 
+// handleUploadFile handles native upload of a file.
+func (b *Bmatrix) handleUploadFile(msg *config.Message, channel string, fi *config.FileInfo) {
+	content := bytes.NewReader(*fi.Data)
+	sp := strings.Split(fi.Name, ".")
+	mtype := mime.TypeByExtension("." + sp[len(sp)-1])
+	if !(strings.Contains(mtype, "image") || strings.Contains(mtype, "video") ||
+		strings.Contains(mtype, "application") || strings.Contains(mtype, "audio")) {
+		return
+	}
+	if fi.Comment != "" {
+		_, err := b.mc.SendText(channel, msg.Username+fi.Comment)
+		if err != nil {
+			b.Log.Errorf("file comment failed: %#v", err)
+		}
+	} else {
+		// image and video uploads send no username, we have to do this ourself here #715
+		_, err := b.mc.SendText(channel, msg.Username)
+		if err != nil {
+			b.Log.Errorf("file comment failed: %#v", err)
+		}
+	}
+	b.Log.Debugf("uploading file: %s %s", fi.Name, mtype)
+	res, err := b.mc.UploadToContentRepo(content, mtype, int64(len(*fi.Data)))
+	if err != nil {
+		b.Log.Errorf("file upload failed: %#v", err)
+		return
+	}
+
+	switch {
+	case strings.Contains(mtype, "video"):
+		b.Log.Debugf("sendVideo %s", res.ContentURI)
+		_, err = b.mc.SendVideo(channel, fi.Name, res.ContentURI)
+		if err != nil {
+			b.Log.Errorf("sendVideo failed: %#v", err)
+		}
+	case strings.Contains(mtype, "image"):
+		b.Log.Debugf("sendImage %s", res.ContentURI)
+		_, err = b.mc.SendImage(channel, fi.Name, res.ContentURI)
+		if err != nil {
+			b.Log.Errorf("sendImage failed: %#v", err)
+		}
+	case strings.Contains(mtype, "application"):
+		b.Log.Debugf("sendFile %s", res.ContentURI)
+		_, err = b.mc.SendMessageEvent(channel, "m.room.message", matrix.FileMessage{
+			MsgType: "m.file",
+			Body:    fi.Name,
+			URL:     res.ContentURI,
+			Info: matrix.FileInfo{
+				Mimetype: mtype,
+				Size:     uint(len(*fi.Data)),
+			},
+		})
+		if err != nil {
+			b.Log.Errorf("sendFile failed: %#v", err)
+		}
+	case strings.Contains(mtype, "audio"):
+		b.Log.Debugf("sendAudio %s", res.ContentURI)
+		_, err = b.mc.SendMessageEvent(channel, "m.room.message", matrix.AudioMessage{
+			MsgType: "m.audio",
+			Body:    fi.Name,
+			URL:     res.ContentURI,
+			Info: matrix.AudioInfo{
+				Mimetype: mtype,
+				Size:     uint(len(*fi.Data)),
+			},
+		})
+		if err != nil {
+			b.Log.Errorf("sendAudio failed: %#v", err)
+		}
+	}
+	b.Log.Debugf("result: %#v", res)
+}
+
 // skipMessages returns true if this message should not be handled
 func (b *Bmatrix) containsAttachment(content map[string]interface{}) bool {
 	// Skip empty messages
@@ -310,3 +416,42 @@ func (b *Bmatrix) containsAttachment(content map[string]interface{}) bool {
 	}
 	return true
 }
+
+// getAvatarURL returns the avatar URL of the specified sender
+func (b *Bmatrix) getAvatarURL(sender string) string {
+	urlPath := b.mc.BuildURL("profile", sender, "avatar_url")
+
+	s := struct {
+		AvatarURL string `json:"avatar_url"`
+	}{}
+
+	err := b.mc.MakeRequest("GET", urlPath, nil, &s)
+	if err != nil {
+		b.Log.Errorf("getAvatarURL failed: %s", err)
+		return ""
+	}
+	url := strings.ReplaceAll(s.AvatarURL, "mxc://", b.GetString("Server")+"/_matrix/media/r0/thumbnail/")
+	if url != "" {
+		url += "?width=37&height=37&method=crop"
+	}
+	return url
+}
+
+func handleError(err error) *httpError {
+	mErr, ok := err.(matrix.HTTPError)
+	if !ok {
+		return &httpError{
+			Err: "not a HTTPError",
+		}
+	}
+
+	var httpErr httpError
+
+	if err := json.Unmarshal(mErr.Contents, &httpErr); err != nil {
+		return &httpError{
+			Err: "unmarshal failed",
+		}
+	}
+
+	return &httpErr
+}

bridge/mattermost/handlers.go

@@ -0,0 +1,199 @@
+package bmattermost
+
+import (
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/42wim/matterbridge/bridge/helper"
+	"github.com/42wim/matterbridge/matterclient"
+	"github.com/mattermost/mattermost-server/v5/model"
+)
+
+// handleDownloadAvatar downloads the avatar of userid from channel
+// sends a EVENT_AVATAR_DOWNLOAD message to the gateway if successful.
+// logs an error message if it fails
+func (b *Bmattermost) handleDownloadAvatar(userid string, channel string) {
+	rmsg := config.Message{
+		Username: "system",
+		Text:     "avatar",
+		Channel:  channel,
+		Account:  b.Account,
+		UserID:   userid,
+		Event:    config.EventAvatarDownload,
+		Extra:    make(map[string][]interface{}),
+	}
+	if _, ok := b.avatarMap[userid]; !ok {
+		data, resp := b.mc.Client.GetProfileImage(userid, "")
+		if resp.Error != nil {
+			b.Log.Errorf("ProfileImage download failed for %#v %s", userid, resp.Error)
+			return
+		}
+		err := helper.HandleDownloadSize(b.Log, &rmsg, userid+".png", int64(len(data)), b.General)
+		if err != nil {
+			b.Log.Error(err)
+			return
+		}
+		helper.HandleDownloadData(b.Log, &rmsg, userid+".png", rmsg.Text, "", &data, b.General)
+		b.Remote <- rmsg
+	}
+}
+
+// handleDownloadFile handles file download
+func (b *Bmattermost) handleDownloadFile(rmsg *config.Message, id string) error {
+	url, _ := b.mc.Client.GetFileLink(id)
+	finfo, resp := b.mc.Client.GetFileInfo(id)
+	if resp.Error != nil {
+		return resp.Error
+	}
+	err := helper.HandleDownloadSize(b.Log, rmsg, finfo.Name, finfo.Size, b.General)
+	if err != nil {
+		return err
+	}
+	data, resp := b.mc.Client.DownloadFile(id, true)
+	if resp.Error != nil {
+		return resp.Error
+	}
+	helper.HandleDownloadData(b.Log, rmsg, finfo.Name, rmsg.Text, url, &data, b.General)
+	return nil
+}
+
+func (b *Bmattermost) handleMatter() {
+	messages := make(chan *config.Message)
+	if b.GetString("WebhookBindAddress") != "" {
+		b.Log.Debugf("Choosing webhooks based receiving")
+		go b.handleMatterHook(messages)
+	} else {
+		if b.GetString("Token") != "" {
+			b.Log.Debugf("Choosing token based receiving")
+		} else {
+			b.Log.Debugf("Choosing login/password based receiving")
+		}
+		// if for some reason we only want to sent stuff to mattermost but not receive, return
+		if b.GetString("WebhookBindAddress") == "" && b.GetString("WebhookURL") != "" && b.GetString("Token") == "" && b.GetString("Login") == "" {
+			b.Log.Debugf("No WebhookBindAddress specified, only WebhookURL. You will not receive messages from mattermost, only sending is possible.")
+		}
+		go b.handleMatterClient(messages)
+	}
+	var ok bool
+	for message := range messages {
+		message.Avatar = helper.GetAvatar(b.avatarMap, message.UserID, b.General)
+		message.Account = b.Account
+		message.Text, ok = b.replaceAction(message.Text)
+		if ok {
+			message.Event = config.EventUserAction
+		}
+		b.Log.Debugf("<= Sending message from %s on %s to gateway", message.Username, b.Account)
+		b.Log.Debugf("<= Message is %#v", message)
+		b.Remote <- *message
+	}
+}
+
+func (b *Bmattermost) handleMatterClient(messages chan *config.Message) {
+	for message := range b.mc.MessageChan {
+		b.Log.Debugf("%#v", message.Raw.Data)
+
+		if b.skipMessage(message) {
+			b.Log.Debugf("Skipped message: %#v", message)
+			continue
+		}
+
+		// only download avatars if we have a place to upload them (configured mediaserver)
+		if b.General.MediaServerUpload != "" || b.General.MediaDownloadPath != "" {
+			b.handleDownloadAvatar(message.UserID, message.Channel)
+		}
+
+		b.Log.Debugf("== Receiving event %#v", message)
+
+		rmsg := &config.Message{
+			Username: message.Username,
+			UserID:   message.UserID,
+			Channel:  message.Channel,
+			Text:     message.Text,
+			ID:       message.Post.Id,
+			ParentID: message.Post.ParentId,
+			Extra:    make(map[string][]interface{}),
+		}
+
+		// handle mattermost post properties (override username and attachments)
+		b.handleProps(rmsg, message)
+
+		// create a text for bridges that don't support native editing
+		if message.Raw.Event == model.WEBSOCKET_EVENT_POST_EDITED && !b.GetBool("EditDisable") {
+			rmsg.Text = message.Text + b.GetString("EditSuffix")
+		}
+
+		if message.Raw.Event == model.WEBSOCKET_EVENT_POST_DELETED {
+			rmsg.Event = config.EventMsgDelete
+		}
+
+		for _, id := range message.Post.FileIds {
+			err := b.handleDownloadFile(rmsg, id)
+			if err != nil {
+				b.Log.Errorf("download failed: %s", err)
+			}
+		}
+
+		// Use nickname instead of username if defined
+		if nick := b.mc.GetNickName(rmsg.UserID); nick != "" {
+			rmsg.Username = nick
+		}
+
+		messages <- rmsg
+	}
+}
+
+func (b *Bmattermost) handleMatterHook(messages chan *config.Message) {
+	for {
+		message := b.mh.Receive()
+		b.Log.Debugf("Receiving from matterhook %#v", message)
+		messages <- &config.Message{
+			UserID:   message.UserID,
+			Username: message.UserName,
+			Text:     message.Text,
+			Channel:  message.ChannelName,
+		}
+	}
+}
+
+// handleUploadFile handles native upload of files
+func (b *Bmattermost) handleUploadFile(msg *config.Message) (string, error) {
+	var err error
+	var res, id string
+	channelID := b.mc.GetChannelId(msg.Channel, b.TeamID)
+	for _, f := range msg.Extra["file"] {
+		fi := f.(config.FileInfo)
+		id, err = b.mc.UploadFile(*fi.Data, channelID, fi.Name)
+		if err != nil {
+			return "", err
+		}
+		msg.Text = fi.Comment
+		if b.GetBool("PrefixMessagesWithNick") {
+			msg.Text = msg.Username + msg.Text
+		}
+		res, err = b.mc.PostMessageWithFiles(channelID, msg.Text, msg.ParentID, []string{id})
+	}
+	return res, err
+}
+
+func (b *Bmattermost) handleProps(rmsg *config.Message, message *matterclient.Message) {
+	props := message.Post.Props
+	if props == nil {
+		return
+	}
+	if _, ok := props["override_username"].(string); ok {
+		rmsg.Username = props["override_username"].(string)
+	}
+	if _, ok := props["attachments"].([]interface{}); ok {
+		rmsg.Extra["attachments"] = props["attachments"].([]interface{})
+		if rmsg.Text == "" {
+			for _, attachment := range rmsg.Extra["attachments"] {
+				attach := attachment.(map[string]interface{})
+				if attach["text"].(string) != "" {
+					rmsg.Text += attach["text"].(string)
+					continue
+				}
+				if attach["fallback"].(string) != "" {
+					rmsg.Text += attach["fallback"].(string)
+				}
+			}
+		}
+	}
+}

bridge/mattermost/helpers.go

@@ -0,0 +1,225 @@
+package bmattermost
+
+import (
+	"strings"
+
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/42wim/matterbridge/bridge/helper"
+	"github.com/42wim/matterbridge/matterclient"
+	"github.com/42wim/matterbridge/matterhook"
+	"github.com/mattermost/mattermost-server/v5/model"
+)
+
+func (b *Bmattermost) doConnectWebhookBind() error {
+	switch {
+	case b.GetString("WebhookURL") != "":
+		b.Log.Info("Connecting using webhookurl (sending) and webhookbindaddress (receiving)")
+		b.mh = matterhook.New(b.GetString("WebhookURL"),
+			matterhook.Config{InsecureSkipVerify: b.GetBool("SkipTLSVerify"),
+				BindAddress: b.GetString("WebhookBindAddress")})
+	case b.GetString("Token") != "":
+		b.Log.Info("Connecting using token (sending)")
+		err := b.apiLogin()
+		if err != nil {
+			return err
+		}
+	case b.GetString("Login") != "":
+		b.Log.Info("Connecting using login/password (sending)")
+		err := b.apiLogin()
+		if err != nil {
+			return err
+		}
+	default:
+		b.Log.Info("Connecting using webhookbindaddress (receiving)")
+		b.mh = matterhook.New(b.GetString("WebhookURL"),
+			matterhook.Config{InsecureSkipVerify: b.GetBool("SkipTLSVerify"),
+				BindAddress: b.GetString("WebhookBindAddress")})
+	}
+	return nil
+}
+
+func (b *Bmattermost) doConnectWebhookURL() error {
+	b.Log.Info("Connecting using webhookurl (sending)")
+	b.mh = matterhook.New(b.GetString("WebhookURL"),
+		matterhook.Config{InsecureSkipVerify: b.GetBool("SkipTLSVerify"),
+			DisableServer: true})
+	if b.GetString("Token") != "" {
+		b.Log.Info("Connecting using token (receiving)")
+		err := b.apiLogin()
+		if err != nil {
+			return err
+		}
+	} else if b.GetString("Login") != "" {
+		b.Log.Info("Connecting using login/password (receiving)")
+		err := b.apiLogin()
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (b *Bmattermost) apiLogin() error {
+	password := b.GetString("Password")
+	if b.GetString("Token") != "" {
+		password = "token=" + b.GetString("Token")
+	}
+
+	b.mc = matterclient.New(b.GetString("Login"), password, b.GetString("Team"), b.GetString("Server"))
+	if b.GetBool("debug") {
+		b.mc.SetLogLevel("debug")
+	}
+	b.mc.SkipTLSVerify = b.GetBool("SkipTLSVerify")
+	b.mc.SkipVersionCheck = b.GetBool("SkipVersionCheck")
+	b.mc.NoTLS = b.GetBool("NoTLS")
+	b.Log.Infof("Connecting %s (team: %s) on %s", b.GetString("Login"), b.GetString("Team"), b.GetString("Server"))
+	err := b.mc.Login()
+	if err != nil {
+		return err
+	}
+	b.Log.Info("Connection succeeded")
+	b.TeamID = b.mc.GetTeamId()
+	go b.mc.WsReceiver()
+	go b.mc.StatusLoop()
+	return nil
+}
+
+// replaceAction replace the message with the correct action (/me) code
+func (b *Bmattermost) replaceAction(text string) (string, bool) {
+	if strings.HasPrefix(text, "*") && strings.HasSuffix(text, "*") {
+		return strings.Replace(text, "*", "", -1), true
+	}
+	return text, false
+}
+
+func (b *Bmattermost) cacheAvatar(msg *config.Message) (string, error) {
+	fi := msg.Extra["file"][0].(config.FileInfo)
+	/* if we have a sha we have successfully uploaded the file to the media server,
+	so we can now cache the sha */
+	if fi.SHA != "" {
+		b.Log.Debugf("Added %s to %s in avatarMap", fi.SHA, msg.UserID)
+		b.avatarMap[msg.UserID] = fi.SHA
+	}
+	return "", nil
+}
+
+// sendWebhook uses the configured WebhookURL to send the message
+func (b *Bmattermost) sendWebhook(msg config.Message) (string, error) {
+	// skip events
+	if msg.Event != "" {
+		return "", nil
+	}
+
+	if b.GetBool("PrefixMessagesWithNick") {
+		msg.Text = msg.Username + msg.Text
+	}
+	if msg.Extra != nil {
+		// this sends a message only if we received a config.EVENT_FILE_FAILURE_SIZE
+		for _, rmsg := range helper.HandleExtra(&msg, b.General) {
+			rmsg := rmsg // scopelint
+			iconURL := config.GetIconURL(&rmsg, b.GetString("iconurl"))
+			matterMessage := matterhook.OMessage{
+				IconURL:  iconURL,
+				Channel:  rmsg.Channel,
+				UserName: rmsg.Username,
+				Text:     rmsg.Text,
+				Props:    make(map[string]interface{}),
+			}
+			matterMessage.Props["matterbridge_"+b.uuid] = true
+			if err := b.mh.Send(matterMessage); err != nil {
+				b.Log.Errorf("sendWebhook failed: %s ", err)
+			}
+		}
+
+		// webhook doesn't support file uploads, so we add the url manually
+		if len(msg.Extra["file"]) > 0 {
+			for _, f := range msg.Extra["file"] {
+				fi := f.(config.FileInfo)
+				if fi.URL != "" {
+					msg.Text += fi.URL
+				}
+			}
+		}
+	}
+
+	iconURL := config.GetIconURL(&msg, b.GetString("iconurl"))
+	matterMessage := matterhook.OMessage{
+		IconURL:  iconURL,
+		Channel:  msg.Channel,
+		UserName: msg.Username,
+		Text:     msg.Text,
+		Props:    make(map[string]interface{}),
+	}
+	if msg.Avatar != "" {
+		matterMessage.IconURL = msg.Avatar
+	}
+	matterMessage.Props["matterbridge_"+b.uuid] = true
+	err := b.mh.Send(matterMessage)
+	if err != nil {
+		b.Log.Info(err)
+		return "", err
+	}
+	return "", nil
+}
+
+// skipMessages returns true if this message should not be handled
+func (b *Bmattermost) skipMessage(message *matterclient.Message) bool {
+	// Handle join/leave
+	if message.Type == "system_join_leave" ||
+		message.Type == "system_join_channel" ||
+		message.Type == "system_leave_channel" {
+		if b.GetBool("nosendjoinpart") {
+			return true
+		}
+		b.Log.Debugf("Sending JOIN_LEAVE event from %s to gateway", b.Account)
+		b.Remote <- config.Message{
+			Username: "system",
+			Text:     message.Text,
+			Channel:  message.Channel,
+			Account:  b.Account,
+			Event:    config.EventJoinLeave,
+		}
+		return true
+	}
+
+	// Handle edited messages
+	if (message.Raw.Event == model.WEBSOCKET_EVENT_POST_EDITED) && b.GetBool("EditDisable") {
+		return true
+	}
+
+	// Ignore non-post messages
+	if message.Post == nil {
+		b.Log.Debugf("ignoring nil message.Post: %#v", message)
+		return true
+	}
+
+	// Ignore messages sent from matterbridge
+	if message.Post.Props != nil {
+		if _, ok := message.Post.Props["matterbridge_"+b.uuid].(bool); ok {
+			b.Log.Debugf("sent by matterbridge, ignoring")
+			return true
+		}
+	}
+
+	// Ignore messages sent from a user logged in as the bot
+	if b.mc.User.Username == message.Username {
+		return true
+	}
+
+	// if the message has reactions don't repost it (for now, until we can correlate reaction with message)
+	if message.Post.HasReactions {
+		return true
+	}
+
+	// ignore messages from other teams than ours
+	if message.Raw.Data["team_id"].(string) != b.TeamID {
+		return true
+	}
+
+	// only handle posted, edited or deleted events
+	if !(message.Raw.Event == "posted" || message.Raw.Event == model.WEBSOCKET_EVENT_POST_EDITED ||
+		message.Raw.Event == model.WEBSOCKET_EVENT_POST_DELETED) {
+		return true
+	}
+	return false
+}

bridge/mattermost/mattermost.go

@@ -3,7 +3,6 @@ package bmattermost
 import (
 	"errors"
 	"fmt"
-	"strings"
 
 	"github.com/42wim/matterbridge/bridge"
 	"github.com/42wim/matterbridge/bridge/config"
@@ -22,6 +21,8 @@ type Bmattermost struct {
 	avatarMap map[string]string
 }
 
+const mattermostPlugin = "mattermost.plugin"
+
 func New(cfg *bridge.Config) bridge.Bridger {
 	b := &Bmattermost{Config: cfg, avatarMap: make(map[string]string)}
 	b.uuid = xid.New().String()
@@ -33,62 +34,31 @@ func (b *Bmattermost) Command(cmd string) string {
 }
 
 func (b *Bmattermost) Connect() error {
+	if b.Account == mattermostPlugin {
+		return nil
+	}
 	if b.GetString("WebhookBindAddress") != "" {
-		if b.GetString("WebhookURL") != "" {
+		if err := b.doConnectWebhookBind(); err != nil {
-			b.Log.Info("Connecting using webhookurl (sending) and webhookbindaddress (receiving)")
+			return err
-			b.mh = matterhook.New(b.GetString("WebhookURL"),
-				matterhook.Config{InsecureSkipVerify: b.GetBool("SkipTLSVerify"),
-					BindAddress: b.GetString("WebhookBindAddress")})
-		} else if b.GetString("Token") != "" {
-			b.Log.Info("Connecting using token (sending)")
-			err := b.apiLogin()
-			if err != nil {
-				return err
-			}
-		} else if b.GetString("Login") != "" {
-			b.Log.Info("Connecting using login/password (sending)")
-			err := b.apiLogin()
-			if err != nil {
-				return err
-			}
-		} else {
-			b.Log.Info("Connecting using webhookbindaddress (receiving)")
-			b.mh = matterhook.New(b.GetString("WebhookURL"),
-				matterhook.Config{InsecureSkipVerify: b.GetBool("SkipTLSVerify"),
-					BindAddress: b.GetString("WebhookBindAddress")})
 		}
 		go b.handleMatter()
 		return nil
 	}
-	if b.GetString("WebhookURL") != "" {
+	switch {
-		b.Log.Info("Connecting using webhookurl (sending)")
+	case b.GetString("WebhookURL") != "":
-		b.mh = matterhook.New(b.GetString("WebhookURL"),
+		if err := b.doConnectWebhookURL(); err != nil {
-			matterhook.Config{InsecureSkipVerify: b.GetBool("SkipTLSVerify"),
+			return err
-				DisableServer: true})
-		if b.GetString("Token") != "" {
-			b.Log.Info("Connecting using token (receiving)")
-			err := b.apiLogin()
-			if err != nil {
-				return err
-			}
-			go b.handleMatter()
-		} else if b.GetString("Login") != "" {
-			b.Log.Info("Connecting using login/password (receiving)")
-			err := b.apiLogin()
-			if err != nil {
-				return err
-			}
-			go b.handleMatter()
 		}
+		go b.handleMatter()
 		return nil
-	} else if b.GetString("Token") != "" {
+	case b.GetString("Token") != "":
 		b.Log.Info("Connecting using token (sending and receiving)")
 		err := b.apiLogin()
 		if err != nil {
 			return err
 		}
 		go b.handleMatter()
-	} else if b.GetString("Login") != "" {
+	case b.GetString("Login") != "":
 		b.Log.Info("Connecting using login/password (sending and receiving)")
 		err := b.apiLogin()
 		if err != nil {
@@ -96,7 +66,8 @@ func (b *Bmattermost) Connect() error {
 		}
 		go b.handleMatter()
 	}
-	if b.GetString("WebhookBindAddress") == "" && b.GetString("WebhookURL") == "" && b.GetString("Login") == "" && b.GetString("Token") == "" {
+	if b.GetString("WebhookBindAddress") == "" && b.GetString("WebhookURL") == "" &&
+		b.GetString("Login") == "" && b.GetString("Token") == "" {
 		return errors.New("no connection method found. See that you have WebhookBindAddress, WebhookURL or Token/Login/Password/Server/Team configured")
 	}
 	return nil
@@ -107,9 +78,12 @@ func (b *Bmattermost) Disconnect() error {
 }
 
 func (b *Bmattermost) JoinChannel(channel config.ChannelInfo) error {
+	if b.Account == mattermostPlugin {
+		return nil
+	}
 	// we can only join channels using the API
 	if b.GetString("WebhookURL") == "" && b.GetString("WebhookBindAddress") == "" {
-		id := b.mc.GetChannelId(channel.Name, "")
+		id := b.mc.GetChannelId(channel.Name, b.TeamID)
 		if id == "" {
 			return fmt.Errorf("Could not find channel ID for channel %s", channel.Name)
 		}
@@ -119,15 +93,18 @@ func (b *Bmattermost) JoinChannel(channel config.ChannelInfo) error {
 }
 
 func (b *Bmattermost) Send(msg config.Message) (string, error) {
+	if b.Account == mattermostPlugin {
+		return "", nil
+	}
 	b.Log.Debugf("=> Receiving %#v", msg)
 
 	// Make a action /me of the message
-	if msg.Event == config.EVENT_USER_ACTION {
+	if msg.Event == config.EventUserAction {
 		msg.Text = "*" + msg.Text + "*"
 	}
 
 	// map the file SHA to our user (caches the avatar)
-	if msg.Event == config.EVENT_AVATAR_DOWNLOAD {
+	if msg.Event == config.EventAvatarDownload {
 		return b.cacheAvatar(&msg)
 	}
 
@@ -137,17 +114,25 @@ func (b *Bmattermost) Send(msg config.Message) (string, error) {
 	}
 
 	// Delete message
-	if msg.Event == config.EVENT_MSG_DELETE {
+	if msg.Event == config.EventMsgDelete {
 		if msg.ID == "" {
 			return "", nil
 		}
 		return msg.ID, b.mc.DeleteMessage(msg.ID)
 	}
 
+	// Handle prefix hint for unthreaded messages.
+	if msg.ParentID == "msg-parent-not-found" {
+		msg.ParentID = ""
+		msg.Text = fmt.Sprintf("[thread]: %s", msg.Text)
+	}
+
 	// Upload a file if it exists
 	if msg.Extra != nil {
 		for _, rmsg := range helper.HandleExtra(&msg, b.General) {
-			b.mc.PostMessage(b.mc.GetChannelId(rmsg.Channel, ""), rmsg.Username+rmsg.Text)
+			if _, err := b.mc.PostMessage(b.mc.GetChannelId(rmsg.Channel, b.TeamID), rmsg.Username+rmsg.Text, msg.ParentID); err != nil {
+				b.Log.Errorf("PostMessage failed: %s", err)
+			}
 		}
 		if len(msg.Extra["file"]) > 0 {
 			return b.handleUploadFile(&msg)
@@ -165,298 +150,5 @@ func (b *Bmattermost) Send(msg config.Message) (string, error) {
 	}
 
 	// Post normal message
-	return b.mc.PostMessage(b.mc.GetChannelId(msg.Channel, ""), msg.Text)
+	return b.mc.PostMessage(b.mc.GetChannelId(msg.Channel, b.TeamID), msg.Text, msg.ParentID)
-}
-
-func (b *Bmattermost) handleMatter() {
-	messages := make(chan *config.Message)
-	if b.GetString("WebhookBindAddress") != "" {
-		b.Log.Debugf("Choosing webhooks based receiving")
-		go b.handleMatterHook(messages)
-	} else {
-		if b.GetString("Token") != "" {
-			b.Log.Debugf("Choosing token based receiving")
-		} else {
-			b.Log.Debugf("Choosing login/password based receiving")
-		}
-		go b.handleMatterClient(messages)
-	}
-	var ok bool
-	for message := range messages {
-		message.Avatar = helper.GetAvatar(b.avatarMap, message.UserID, b.General)
-		message.Account = b.Account
-		message.Text, ok = b.replaceAction(message.Text)
-		if ok {
-			message.Event = config.EVENT_USER_ACTION
-		}
-		b.Log.Debugf("<= Sending message from %s on %s to gateway", message.Username, b.Account)
-		b.Log.Debugf("<= Message is %#v", message)
-		b.Remote <- *message
-	}
-}
-
-func (b *Bmattermost) handleMatterClient(messages chan *config.Message) {
-	for message := range b.mc.MessageChan {
-		b.Log.Debugf("%#v", message.Raw.Data)
-
-		if b.skipMessage(message) {
-			b.Log.Debugf("Skipped message: %#v", message)
-			continue
-		}
-
-		// only download avatars if we have a place to upload them (configured mediaserver)
-		if b.General.MediaServerUpload != "" {
-			b.handleDownloadAvatar(message.UserID, message.Channel)
-		}
-
-		b.Log.Debugf("== Receiving event %#v", message)
-
-		rmsg := &config.Message{Username: message.Username, UserID: message.UserID, Channel: message.Channel, Text: message.Text, ID: message.Post.Id, Extra: make(map[string][]interface{})}
-
-		// handle mattermost post properties (override username and attachments)
-		props := message.Post.Props
-		if props != nil {
-			if _, ok := props["override_username"].(string); ok {
-				rmsg.Username = props["override_username"].(string)
-			}
-			if _, ok := props["attachments"].([]interface{}); ok {
-				rmsg.Extra["attachments"] = props["attachments"].([]interface{})
-				if rmsg.Text == "" {
-					for _, attachment := range rmsg.Extra["attachments"] {
-						attach := attachment.(map[string]interface{})
-						if attach["text"].(string) != "" {
-							rmsg.Text += attach["text"].(string)
-							continue
-						}
-						if attach["fallback"].(string) != "" {
-							rmsg.Text += attach["fallback"].(string)
-						}
-					}
-				}
-			}
-		}
-
-		// create a text for bridges that don't support native editing
-		if message.Raw.Event == "post_edited" && !b.GetBool("EditDisable") {
-			rmsg.Text = message.Text + b.GetString("EditSuffix")
-		}
-
-		if message.Raw.Event == "post_deleted" {
-			rmsg.Event = config.EVENT_MSG_DELETE
-		}
-
-		if len(message.Post.FileIds) > 0 {
-			for _, id := range message.Post.FileIds {
-				err := b.handleDownloadFile(rmsg, id)
-				if err != nil {
-					b.Log.Errorf("download failed: %s", err)
-				}
-			}
-		}
-		messages <- rmsg
-	}
-}
-
-func (b *Bmattermost) handleMatterHook(messages chan *config.Message) {
-	for {
-		message := b.mh.Receive()
-		b.Log.Debugf("Receiving from matterhook %#v", message)
-		messages <- &config.Message{UserID: message.UserID, Username: message.UserName, Text: message.Text, Channel: message.ChannelName}
-	}
-}
-
-func (b *Bmattermost) apiLogin() error {
-	password := b.GetString("Password")
-	if b.GetString("Token") != "" {
-		password = "MMAUTHTOKEN=" + b.GetString("Token")
-	}
-
-	b.mc = matterclient.New(b.GetString("Login"), password, b.GetString("Team"), b.GetString("Server"))
-	if b.GetBool("debug") {
-		b.mc.SetLogLevel("debug")
-	}
-	b.mc.SkipTLSVerify = b.GetBool("SkipTLSVerify")
-	b.mc.NoTLS = b.GetBool("NoTLS")
-	b.Log.Infof("Connecting %s (team: %s) on %s", b.GetString("Login"), b.GetString("Team"), b.GetString("Server"))
-	err := b.mc.Login()
-	if err != nil {
-		return err
-	}
-	b.Log.Info("Connection succeeded")
-	b.TeamID = b.mc.GetTeamId()
-	go b.mc.WsReceiver()
-	go b.mc.StatusLoop()
-	return nil
-}
-
-// replaceAction replace the message with the correct action (/me) code
-func (b *Bmattermost) replaceAction(text string) (string, bool) {
-	if strings.HasPrefix(text, "*") && strings.HasSuffix(text, "*") {
-		return strings.Replace(text, "*", "", -1), true
-	}
-	return text, false
-}
-
-func (b *Bmattermost) cacheAvatar(msg *config.Message) (string, error) {
-	fi := msg.Extra["file"][0].(config.FileInfo)
-	/* if we have a sha we have successfully uploaded the file to the media server,
-	so we can now cache the sha */
-	if fi.SHA != "" {
-		b.Log.Debugf("Added %s to %s in avatarMap", fi.SHA, msg.UserID)
-		b.avatarMap[msg.UserID] = fi.SHA
-	}
-	return "", nil
-}
-
-// handleDownloadAvatar downloads the avatar of userid from channel
-// sends a EVENT_AVATAR_DOWNLOAD message to the gateway if successful.
-// logs an error message if it fails
-func (b *Bmattermost) handleDownloadAvatar(userid string, channel string) {
-	rmsg := config.Message{Username: "system", Text: "avatar", Channel: channel, Account: b.Account, UserID: userid, Event: config.EVENT_AVATAR_DOWNLOAD, Extra: make(map[string][]interface{})}
-	if _, ok := b.avatarMap[userid]; !ok {
-		data, resp := b.mc.Client.GetProfileImage(userid, "")
-		if resp.Error != nil {
-			b.Log.Errorf("ProfileImage download failed for %#v %s", userid, resp.Error)
-			return
-		}
-		err := helper.HandleDownloadSize(b.Log, &rmsg, userid+".png", int64(len(data)), b.General)
-		if err != nil {
-			b.Log.Error(err)
-			return
-		}
-		helper.HandleDownloadData(b.Log, &rmsg, userid+".png", rmsg.Text, "", &data, b.General)
-		b.Remote <- rmsg
-	}
-}
-
-// handleDownloadFile handles file download
-func (b *Bmattermost) handleDownloadFile(rmsg *config.Message, id string) error {
-	url, _ := b.mc.Client.GetFileLink(id)
-	finfo, resp := b.mc.Client.GetFileInfo(id)
-	if resp.Error != nil {
-		return resp.Error
-	}
-	err := helper.HandleDownloadSize(b.Log, rmsg, finfo.Name, finfo.Size, b.General)
-	if err != nil {
-		return err
-	}
-	data, resp := b.mc.Client.DownloadFile(id, true)
-	if resp.Error != nil {
-		return resp.Error
-	}
-	helper.HandleDownloadData(b.Log, rmsg, finfo.Name, rmsg.Text, url, &data, b.General)
-	return nil
-}
-
-// handleUploadFile handles native upload of files
-func (b *Bmattermost) handleUploadFile(msg *config.Message) (string, error) {
-	var err error
-	var res, id string
-	channelID := b.mc.GetChannelId(msg.Channel, "")
-	for _, f := range msg.Extra["file"] {
-		fi := f.(config.FileInfo)
-		id, err = b.mc.UploadFile(*fi.Data, channelID, fi.Name)
-		if err != nil {
-			return "", err
-		}
-		msg.Text = fi.Comment
-		if b.GetBool("PrefixMessagesWithNick") {
-			msg.Text = msg.Username + msg.Text
-		}
-		res, err = b.mc.PostMessageWithFiles(channelID, msg.Text, []string{id})
-	}
-	return res, err
-}
-
-// sendWebhook uses the configured WebhookURL to send the message
-func (b *Bmattermost) sendWebhook(msg config.Message) (string, error) {
-	// skip events
-	if msg.Event != "" {
-		return "", nil
-	}
-
-	if b.GetBool("PrefixMessagesWithNick") {
-		msg.Text = msg.Username + msg.Text
-	}
-	if msg.Extra != nil {
-		// this sends a message only if we received a config.EVENT_FILE_FAILURE_SIZE
-		for _, rmsg := range helper.HandleExtra(&msg, b.General) {
-			iconURL := config.GetIconURL(&rmsg, b.GetString("iconurl"))
-			matterMessage := matterhook.OMessage{IconURL: iconURL, Channel: rmsg.Channel, UserName: rmsg.Username, Text: rmsg.Text, Props: make(map[string]interface{})}
-			matterMessage.Props["matterbridge_"+b.uuid] = true
-			b.mh.Send(matterMessage)
-		}
-
-		// webhook doesn't support file uploads, so we add the url manually
-		if len(msg.Extra["file"]) > 0 {
-			for _, f := range msg.Extra["file"] {
-				fi := f.(config.FileInfo)
-				if fi.URL != "" {
-					msg.Text += fi.URL
-				}
-			}
-		}
-	}
-
-	iconURL := config.GetIconURL(&msg, b.GetString("iconurl"))
-	matterMessage := matterhook.OMessage{IconURL: iconURL, Channel: msg.Channel, UserName: msg.Username, Text: msg.Text, Props: make(map[string]interface{})}
-	if msg.Avatar != "" {
-		matterMessage.IconURL = msg.Avatar
-	}
-	matterMessage.Props["matterbridge_"+b.uuid] = true
-	err := b.mh.Send(matterMessage)
-	if err != nil {
-		b.Log.Info(err)
-		return "", err
-	}
-	return "", nil
-}
-
-// skipMessages returns true if this message should not be handled
-func (b *Bmattermost) skipMessage(message *matterclient.Message) bool {
-	// Handle join/leave
-	if message.Type == "system_join_leave" ||
-		message.Type == "system_join_channel" ||
-		message.Type == "system_leave_channel" {
-		if b.GetBool("nosendjoinpart") {
-			return true
-		}
-		b.Log.Debugf("Sending JOIN_LEAVE event from %s to gateway", b.Account)
-		b.Remote <- config.Message{Username: "system", Text: message.Text, Channel: message.Channel, Account: b.Account, Event: config.EVENT_JOIN_LEAVE}
-		return true
-	}
-
-	// Handle edited messages
-	if (message.Raw.Event == "post_edited") && b.GetBool("EditDisable") {
-		return true
-	}
-
-	// Ignore messages sent from matterbridge
-	if message.Post.Props != nil {
-		if _, ok := message.Post.Props["matterbridge_"+b.uuid].(bool); ok {
-			b.Log.Debugf("sent by matterbridge, ignoring")
-			return true
-		}
-	}
-
-	// Ignore messages sent from a user logged in as the bot
-	if b.mc.User.Username == message.Username {
-		return true
-	}
-
-	// if the message has reactions don't repost it (for now, until we can correlate reaction with message)
-	if message.Post.HasReactions {
-		return true
-	}
-
-	// ignore messages from other teams than ours
-	if message.Raw.Data["team_id"].(string) != b.TeamID {
-		return true
-	}
-
-	// only handle posted, edited or deleted events
-	if !(message.Raw.Event == "posted" || message.Raw.Event == "post_edited" || message.Raw.Event == "post_deleted") {
-		return true
-	}
-	return false
 }

bridge/msteams/handler.go

@@ -0,0 +1,101 @@
+package bmsteams
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"strings"
+
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/42wim/matterbridge/bridge/helper"
+
+	msgraph "github.com/yaegashi/msgraph.go/beta"
+)
+
+func (b *Bmsteams) findFile(weburl string) (string, error) {
+	itemRB, err := b.gc.GetDriveItemByURL(b.ctx, weburl)
+	if err != nil {
+		return "", err
+	}
+	itemRB.Workbook().Worksheets()
+	b.gc.Workbooks()
+	item, err := itemRB.Request().Get(b.ctx)
+	if err != nil {
+		return "", err
+	}
+	if url, ok := item.GetAdditionalData("@microsoft.graph.downloadUrl"); ok {
+		return url.(string), nil
+	}
+	return "", nil
+}
+
+// handleDownloadFile handles file download
+func (b *Bmsteams) handleDownloadFile(rmsg *config.Message, filename, weburl string) error {
+	realURL, err := b.findFile(weburl)
+	if err != nil {
+		return err
+	}
+	// Actually download the file.
+	data, err := helper.DownloadFile(realURL)
+	if err != nil {
+		return fmt.Errorf("download %s failed %#v", weburl, err)
+	}
+
+	// If a comment is attached to the file(s) it is in the 'Text' field of the teams messge event
+	// and should be added as comment to only one of the files. We reset the 'Text' field to ensure
+	// that the comment is not duplicated.
+	comment := rmsg.Text
+	rmsg.Text = ""
+	helper.HandleDownloadData(b.Log, rmsg, filename, comment, weburl, data, b.General)
+	return nil
+}
+
+func (b *Bmsteams) handleAttachments(rmsg *config.Message, msg msgraph.ChatMessage) {
+	for _, a := range msg.Attachments {
+		//remove the attachment tags from the text
+		rmsg.Text = attachRE.ReplaceAllString(rmsg.Text, "")
+
+		//handle a code snippet (code block)
+		if *a.ContentType == "application/vnd.microsoft.card.codesnippet" {
+			b.handleCodeSnippet(rmsg, a)
+			continue
+		}
+
+		//handle the download
+		err := b.handleDownloadFile(rmsg, *a.Name, *a.ContentURL)
+		if err != nil {
+			b.Log.Errorf("download of %s failed: %s", *a.Name, err)
+		}
+	}
+}
+
+type AttachContent struct {
+	Language       string `json:"language"`
+	CodeSnippetURL string `json:"codeSnippetUrl"`
+}
+
+func (b *Bmsteams) handleCodeSnippet(rmsg *config.Message, attach msgraph.ChatMessageAttachment) {
+	var content AttachContent
+	err := json.Unmarshal([]byte(*attach.Content), &content)
+	if err != nil {
+		b.Log.Errorf("unmarshal codesnippet failed: %s", err)
+		return
+	}
+	s := strings.Split(content.CodeSnippetURL, "/")
+	if len(s) != 13 {
+		b.Log.Errorf("codesnippetUrl has unexpected size: %s", content.CodeSnippetURL)
+		return
+	}
+	resp, err := b.gc.Teams().Request().Client().Get(content.CodeSnippetURL)
+	if err != nil {
+		b.Log.Errorf("retrieving snippet content failed:%s", err)
+		return
+	}
+	defer resp.Body.Close()
+	res, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		b.Log.Errorf("reading snippet data failed: %s", err)
+		return
+	}
+	rmsg.Text = rmsg.Text + "\n```" + content.Language + "\n" + string(res) + "\n```\n"
+}

bridge/msteams/msteams.go

@@ -0,0 +1,224 @@
+package bmsteams
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/42wim/matterbridge/bridge"
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/davecgh/go-spew/spew"
+
+	"github.com/mattn/godown"
+	msgraph "github.com/yaegashi/msgraph.go/beta"
+	"github.com/yaegashi/msgraph.go/msauth"
+
+	"golang.org/x/oauth2"
+)
+
+var defaultScopes = []string{"openid", "profile", "offline_access", "Group.Read.All", "Group.ReadWrite.All"}
+var attachRE = regexp.MustCompile(`<attachment id=.*?attachment>`)
+
+type Bmsteams struct {
+	gc    *msgraph.GraphServiceRequestBuilder
+	ctx   context.Context
+	botID string
+	*bridge.Config
+}
+
+func New(cfg *bridge.Config) bridge.Bridger {
+	return &Bmsteams{Config: cfg}
+}
+
+func (b *Bmsteams) Connect() error {
+	tokenCachePath := b.GetString("sessionFile")
+	if tokenCachePath == "" {
+		tokenCachePath = "msteams_session.json"
+	}
+	ctx := context.Background()
+	m := msauth.NewManager()
+	m.LoadFile(tokenCachePath) //nolint:errcheck
+	ts, err := m.DeviceAuthorizationGrant(ctx, b.GetString("TenantID"), b.GetString("ClientID"), defaultScopes, nil)
+	if err != nil {
+		return err
+	}
+	err = m.SaveFile(tokenCachePath)
+	if err != nil {
+		b.Log.Errorf("Couldn't save sessionfile in %s: %s", tokenCachePath, err)
+	}
+	// make file readable only for matterbridge user
+	err = os.Chmod(tokenCachePath, 0600)
+	if err != nil {
+		b.Log.Errorf("Couldn't change permissions for %s: %s", tokenCachePath, err)
+	}
+	httpClient := oauth2.NewClient(ctx, ts)
+	graphClient := msgraph.NewClient(httpClient)
+	b.gc = graphClient
+	b.ctx = ctx
+
+	err = b.setBotID()
+	if err != nil {
+		return err
+	}
+	b.Log.Info("Connection succeeded")
+	return nil
+}
+
+func (b *Bmsteams) Disconnect() error {
+	return nil
+}
+
+func (b *Bmsteams) JoinChannel(channel config.ChannelInfo) error {
+	go func(name string) {
+		for {
+			err := b.poll(name)
+			if err != nil {
+				b.Log.Errorf("polling failed for %s: %s. retrying in 5 seconds", name, err)
+			}
+			time.Sleep(time.Second * 5)
+		}
+	}(channel.Name)
+	return nil
+}
+
+func (b *Bmsteams) Send(msg config.Message) (string, error) {
+	b.Log.Debugf("=> Receiving %#v", msg)
+	if msg.ParentID != "" && msg.ParentID != "msg-parent-not-found" {
+		return b.sendReply(msg)
+	}
+	if msg.ParentID == "msg-parent-not-found" {
+		msg.ParentID = ""
+		msg.Text = fmt.Sprintf("[thread]: %s", msg.Text)
+	}
+	ct := b.gc.Teams().ID(b.GetString("TeamID")).Channels().ID(msg.Channel).Messages().Request()
+	text := msg.Username + msg.Text
+	content := &msgraph.ItemBody{Content: &text}
+	rmsg := &msgraph.ChatMessage{Body: content}
+	res, err := ct.Add(b.ctx, rmsg)
+	if err != nil {
+		return "", err
+	}
+	return *res.ID, nil
+}
+
+func (b *Bmsteams) sendReply(msg config.Message) (string, error) {
+	ct := b.gc.Teams().ID(b.GetString("TeamID")).Channels().ID(msg.Channel).Messages().ID(msg.ParentID).Replies().Request()
+	// Handle prefix hint for unthreaded messages.
+
+	text := msg.Username + msg.Text
+	content := &msgraph.ItemBody{Content: &text}
+	rmsg := &msgraph.ChatMessage{Body: content}
+	res, err := ct.Add(b.ctx, rmsg)
+	if err != nil {
+		return "", err
+	}
+	return *res.ID, nil
+}
+
+func (b *Bmsteams) getMessages(channel string) ([]msgraph.ChatMessage, error) {
+	ct := b.gc.Teams().ID(b.GetString("TeamID")).Channels().ID(channel).Messages().Request()
+	rct, err := ct.Get(b.ctx)
+	if err != nil {
+		return nil, err
+	}
+	b.Log.Debugf("got %#v messages", len(rct))
+	return rct, nil
+}
+
+//nolint:gocognit
+func (b *Bmsteams) poll(channelName string) error {
+	msgmap := make(map[string]time.Time)
+	b.Log.Debug("getting initial messages")
+	res, err := b.getMessages(channelName)
+	if err != nil {
+		return err
+	}
+	for _, msg := range res {
+		msgmap[*msg.ID] = *msg.CreatedDateTime
+		if msg.LastModifiedDateTime != nil {
+			msgmap[*msg.ID] = *msg.LastModifiedDateTime
+		}
+	}
+	time.Sleep(time.Second * 5)
+	b.Log.Debug("polling for messages")
+	for {
+		res, err := b.getMessages(channelName)
+		if err != nil {
+			return err
+		}
+		for i := len(res) - 1; i >= 0; i-- {
+			msg := res[i]
+			if mtime, ok := msgmap[*msg.ID]; ok {
+				if mtime == *msg.CreatedDateTime && msg.LastModifiedDateTime == nil {
+					continue
+				}
+				if msg.LastModifiedDateTime != nil && mtime == *msg.LastModifiedDateTime {
+					continue
+				}
+			}
+
+			if b.GetBool("debug") {
+				b.Log.Debug("Msg dump: ", spew.Sdump(msg))
+			}
+
+			// skip non-user message for now.
+			if msg.From.User == nil {
+				continue
+			}
+
+			if *msg.From.User.ID == b.botID {
+				b.Log.Debug("skipping own message")
+				msgmap[*msg.ID] = *msg.CreatedDateTime
+				continue
+			}
+
+			msgmap[*msg.ID] = *msg.CreatedDateTime
+			if msg.LastModifiedDateTime != nil {
+				msgmap[*msg.ID] = *msg.LastModifiedDateTime
+			}
+			b.Log.Debugf("<= Sending message from %s on %s to gateway", *msg.From.User.DisplayName, b.Account)
+			text := b.convertToMD(*msg.Body.Content)
+			rmsg := config.Message{
+				Username: *msg.From.User.DisplayName,
+				Text:     text,
+				Channel:  channelName,
+				Account:  b.Account,
+				Avatar:   "",
+				UserID:   *msg.From.User.ID,
+				ID:       *msg.ID,
+				Extra:    make(map[string][]interface{}),
+			}
+
+			b.handleAttachments(&rmsg, msg)
+			b.Log.Debugf("<= Message is %#v", rmsg)
+			b.Remote <- rmsg
+		}
+		time.Sleep(time.Second * 5)
+	}
+}
+
+func (b *Bmsteams) setBotID() error {
+	req := b.gc.Me().Request()
+	r, err := req.Get(b.ctx)
+	if err != nil {
+		return err
+	}
+	b.botID = *r.ID
+	return nil
+}
+
+func (b *Bmsteams) convertToMD(text string) string {
+	if !strings.Contains(text, "<div>") {
+		return text
+	}
+	var sb strings.Builder
+	err := godown.Convert(&sb, strings.NewReader(text), nil)
+	if err != nil {
+		b.Log.Errorf("Couldn't convert message to markdown %s", text)
+		return text
+	}
+	return sb.String()
+}

bridge/nctalk/nctalk.go

@@ -0,0 +1,146 @@
+package nctalk
+
+import (
+	"context"
+	"crypto/tls"
+	"strconv"
+	"strings"
+
+	"github.com/42wim/matterbridge/bridge"
+	"github.com/42wim/matterbridge/bridge/config"
+
+	talk "gomod.garykim.dev/nc-talk"
+	"gomod.garykim.dev/nc-talk/ocs"
+	"gomod.garykim.dev/nc-talk/room"
+	"gomod.garykim.dev/nc-talk/user"
+)
+
+type Btalk struct {
+	user  *user.TalkUser
+	rooms []Broom
+	*bridge.Config
+}
+
+func New(cfg *bridge.Config) bridge.Bridger {
+	return &Btalk{Config: cfg}
+}
+
+type Broom struct {
+	room      *room.TalkRoom
+	ctx       context.Context
+	ctxCancel context.CancelFunc
+}
+
+func (b *Btalk) Connect() error {
+	b.Log.Info("Connecting")
+	tconfig := &user.TalkUserConfig{
+		TLSConfig: &tls.Config{
+			InsecureSkipVerify: b.GetBool("SkipTLSVerify"), //nolint:gosec
+		},
+	}
+	var err error
+	b.user, err = user.NewUser(b.GetString("Server"), b.GetString("Login"), b.GetString("Password"), tconfig)
+	if err != nil {
+		b.Log.Error("Config could not be used")
+		return err
+	}
+	_, err = b.user.Capabilities()
+	if err != nil {
+		b.Log.Error("Cannot Connect")
+		return err
+	}
+	b.Log.Info("Connected")
+	return nil
+}
+
+func (b *Btalk) Disconnect() error {
+	for _, r := range b.rooms {
+		r.ctxCancel()
+	}
+	return nil
+}
+
+func (b *Btalk) JoinChannel(channel config.ChannelInfo) error {
+	newRoom := Broom{
+		room: talk.NewRoom(b.user, channel.Name),
+	}
+	newRoom.ctx, newRoom.ctxCancel = context.WithCancel(context.Background())
+	c, err := newRoom.room.ReceiveMessages(newRoom.ctx)
+	if err != nil {
+		return err
+	}
+	b.rooms = append(b.rooms, newRoom)
+	go func() {
+		for msg := range c {
+			// ignore messages that are one of the following
+			// * not a message from a user
+			// * from ourselves
+			if msg.MessageType != ocs.MessageComment || msg.ActorID == b.user.User {
+				continue
+			}
+			remoteMessage := config.Message{
+				Text:     formatRichObjectString(msg.Message, msg.MessageParameters),
+				Channel:  newRoom.room.Token,
+				Username: msg.ActorDisplayName,
+				UserID:   msg.ActorID,
+				Account:  b.Account,
+			}
+			// It is possible for the ID to not be set on older versions of Talk so we only set it if
+			// the ID is not blank
+			if msg.ID != 0 {
+				remoteMessage.ID = strconv.Itoa(msg.ID)
+			}
+			b.Log.Debugf("<= Message is %#v", remoteMessage)
+			b.Remote <- remoteMessage
+		}
+	}()
+	return nil
+}
+
+func (b *Btalk) Send(msg config.Message) (string, error) {
+	r := b.getRoom(msg.Channel)
+	if r == nil {
+		b.Log.Errorf("Could not find room for %v", msg.Channel)
+		return "", nil
+	}
+
+	// Talk currently only supports sending normal messages
+	if msg.Event != "" {
+		return "", nil
+	}
+	sentMessage, err := r.room.SendMessage(msg.Username + msg.Text)
+	if err != nil {
+		b.Log.Errorf("Could not send message to room %v from %v: %v", msg.Channel, msg.Username, err)
+		return "", nil
+	}
+	return strconv.Itoa(sentMessage.ID), nil
+}
+
+func (b *Btalk) getRoom(token string) *Broom {
+	for _, r := range b.rooms {
+		if r.room.Token == token {
+			return &r
+		}
+	}
+	return nil
+}
+
+// Spec: https://github.com/nextcloud/server/issues/1706#issue-182308785
+func formatRichObjectString(message string, parameters map[string]ocs.RichObjectString) string {
+	for id, parameter := range parameters {
+		text := parameter.Name
+
+		switch parameter.Type {
+		case ocs.ROSTypeUser, ocs.ROSTypeGroup:
+			text = "@" + text
+		case ocs.ROSTypeFile:
+			if parameter.Link != "" {
+				text = parameter.Link
+			}
+		}
+
+		message = strings.ReplaceAll(message, "{"+id+"}", text)
+	}
+
+	return message
+}

bridge/rocketchat/handlers.go

@@ -0,0 +1,97 @@
+package brocketchat
+
+import (
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/matterbridge/Rocket.Chat.Go.SDK/models"
+)
+
+func (b *Brocketchat) handleRocket() {
+	messages := make(chan *config.Message)
+	if b.GetString("WebhookBindAddress") != "" {
+		b.Log.Debugf("Choosing webhooks based receiving")
+		go b.handleRocketHook(messages)
+	} else {
+		b.Log.Debugf("Choosing login/password based receiving")
+		go b.handleRocketClient(messages)
+	}
+	for message := range messages {
+		message.Account = b.Account
+		b.Log.Debugf("<= Sending message from %s on %s to gateway", message.Username, b.Account)
+		b.Log.Debugf("<= Message is %#v", message)
+		b.Remote <- *message
+	}
+}
+
+func (b *Brocketchat) handleRocketHook(messages chan *config.Message) {
+	for {
+		message := b.rh.Receive()
+		b.Log.Debugf("Receiving from rockethook %#v", message)
+		// do not loop
+		if message.UserName == b.GetString("Nick") {
+			continue
+		}
+		messages <- &config.Message{
+			UserID:   message.UserID,
+			Username: message.UserName,
+			Text:     message.Text,
+			Channel:  message.ChannelName,
+		}
+	}
+}
+
+func (b *Brocketchat) handleStatusEvent(ev models.Message, rmsg *config.Message) bool {
+	switch ev.Type {
+	case "":
+		// this is a normal message, no processing needed
+		// return true so the message is not dropped
+		return true
+	case sUserJoined, sUserLeft:
+		rmsg.Event = config.EventJoinLeave
+		return true
+	case sRoomChangedTopic:
+		rmsg.Event = config.EventTopicChange
+		return true
+	}
+	b.Log.Debugf("Dropping message with unknown type: %s", ev.Type)
+	return false
+}
+
+func (b *Brocketchat) handleRocketClient(messages chan *config.Message) {
+	for message := range b.messageChan {
+		// skip messages with same ID, apparently messages get duplicated for an unknown reason
+		if _, ok := b.cache.Get(message.ID); ok {
+			continue
+		}
+		b.cache.Add(message.ID, true)
+		b.Log.Debugf("message %#v", message)
+		m := message
+		if b.skipMessage(&m) {
+			b.Log.Debugf("Skipped message: %#v", message)
+			continue
+		}
+
+		rmsg := &config.Message{Text: message.Msg,
+			Username: message.User.UserName,
+			Channel:  b.getChannelName(message.RoomID),
+			Account:  b.Account,
+			UserID:   message.User.ID,
+			ID:       message.ID,
+		}
+
+		// handleStatusEvent returns false if the message should be dropped
+		// in that case it is probably some modification to the channel we do not want to relay
+		if b.handleStatusEvent(m, rmsg) {
+			messages <- rmsg
+		}
+	}
+}
+
+func (b *Brocketchat) handleUploadFile(msg *config.Message) error {
+	for _, f := range msg.Extra["file"] {
+		fi := f.(config.FileInfo)
+		if err := b.uploadFile(&fi, b.getChannelID(msg.Channel)); err != nil {
+			return err
+		}
+	}
+	return nil
+}

bridge/rocketchat/helpers.go

@@ -0,0 +1,201 @@
+package brocketchat
+
+import (
+	"context"
+	"io/ioutil"
+	"mime"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/42wim/matterbridge/bridge/helper"
+	"github.com/42wim/matterbridge/hook/rockethook"
+	"github.com/42wim/matterbridge/matterhook"
+	"github.com/matterbridge/Rocket.Chat.Go.SDK/models"
+	"github.com/matterbridge/Rocket.Chat.Go.SDK/realtime"
+	"github.com/matterbridge/Rocket.Chat.Go.SDK/rest"
+	"github.com/nelsonken/gomf"
+)
+
+func (b *Brocketchat) doConnectWebhookBind() error {
+	switch {
+	case b.GetString("WebhookURL") != "":
+		b.Log.Info("Connecting using webhookurl (sending) and webhookbindaddress (receiving)")
+		b.mh = matterhook.New(b.GetString("WebhookURL"),
+			matterhook.Config{InsecureSkipVerify: b.GetBool("SkipTLSVerify"),
+				DisableServer: true})
+		b.rh = rockethook.New(b.GetString("WebhookURL"), rockethook.Config{BindAddress: b.GetString("WebhookBindAddress")})
+	case b.GetString("Login") != "":
+		b.Log.Info("Connecting using login/password (sending)")
+		err := b.apiLogin()
+		if err != nil {
+			return err
+		}
+	default:
+		b.Log.Info("Connecting using webhookbindaddress (receiving)")
+		b.rh = rockethook.New(b.GetString("WebhookURL"), rockethook.Config{BindAddress: b.GetString("WebhookBindAddress")})
+	}
+	return nil
+}
+
+func (b *Brocketchat) doConnectWebhookURL() error {
+	b.Log.Info("Connecting using webhookurl (sending)")
+	b.mh = matterhook.New(b.GetString("WebhookURL"),
+		matterhook.Config{InsecureSkipVerify: b.GetBool("SkipTLSVerify"),
+			DisableServer: true})
+	if b.GetString("Login") != "" {
+		b.Log.Info("Connecting using login/password (receiving)")
+		err := b.apiLogin()
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (b *Brocketchat) apiLogin() error {
+	b.Log.Debugf("handling apiLogin()")
+	credentials := &models.UserCredentials{Email: b.GetString("login"), Password: b.GetString("password")}
+	if b.GetString("Token") != "" {
+		credentials = &models.UserCredentials{ID: b.GetString("Login"), Token: b.GetString("Token")}
+	}
+	myURL, err := url.Parse(b.GetString("server"))
+	if err != nil {
+		return err
+	}
+	client, err := realtime.NewClient(myURL, b.GetBool("debug"))
+	b.c = client
+	if err != nil {
+		return err
+	}
+	restclient := rest.NewClient(myURL, b.GetBool("debug"))
+	user, err := b.c.Login(credentials)
+	if err != nil {
+		return err
+	}
+	b.user = user
+	b.r = restclient
+	err = b.r.Login(credentials)
+	if err != nil {
+		return err
+	}
+	b.Log.Info("Connection succeeded")
+	return nil
+}
+
+func (b *Brocketchat) getChannelName(id string) string {
+	b.RLock()
+	defer b.RUnlock()
+	if name, ok := b.channelMap[id]; ok {
+		return name
+	}
+	return ""
+}
+
+func (b *Brocketchat) getChannelID(name string) string {
+	b.RLock()
+	defer b.RUnlock()
+	for k, v := range b.channelMap {
+		if v == name || v == "#"+name {
+			return k
+		}
+	}
+	return ""
+}
+
+func (b *Brocketchat) skipMessage(message *models.Message) bool {
+	return message.User.ID == b.user.ID
+}
+
+func (b *Brocketchat) uploadFile(fi *config.FileInfo, channel string) error {
+	fb := gomf.New()
+	if err := fb.WriteField("description", fi.Comment); err != nil {
+		return err
+	}
+	sp := strings.Split(fi.Name, ".")
+	mtype := mime.TypeByExtension("." + sp[len(sp)-1])
+	if !strings.Contains(mtype, "image") && !strings.Contains(mtype, "video") {
+		return nil
+	}
+	if err := fb.WriteFile("file", fi.Name, mtype, *fi.Data); err != nil {
+		return err
+	}
+	req, err := fb.GetHTTPRequest(context.TODO(), b.GetString("server")+"/api/v1/rooms.upload/"+channel)
+	if err != nil {
+		return err
+	}
+	req.Header.Add("X-Auth-Token", b.user.Token)
+	req.Header.Add("X-User-Id", b.user.ID)
+	client := &http.Client{
+		Timeout: time.Second * 5,
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	if resp.StatusCode != 200 {
+		b.Log.Errorf("failed: %#v", string(body))
+	}
+	return nil
+}
+
+// sendWebhook uses the configured WebhookURL to send the message
+func (b *Brocketchat) sendWebhook(msg *config.Message) error {
+	// skip events
+	if msg.Event != "" {
+		return nil
+	}
+
+	if b.GetBool("PrefixMessagesWithNick") {
+		msg.Text = msg.Username + msg.Text
+	}
+	if msg.Extra != nil {
+		// this sends a message only if we received a config.EVENT_FILE_FAILURE_SIZE
+		for _, rmsg := range helper.HandleExtra(msg, b.General) {
+			rmsg := rmsg // scopelint
+			iconURL := config.GetIconURL(&rmsg, b.GetString("iconurl"))
+			matterMessage := matterhook.OMessage{
+				IconURL:  iconURL,
+				Channel:  rmsg.Channel,
+				UserName: rmsg.Username,
+				Text:     rmsg.Text,
+				Props:    make(map[string]interface{}),
+			}
+			if err := b.mh.Send(matterMessage); err != nil {
+				b.Log.Errorf("sendWebhook failed: %s ", err)
+			}
+		}
+
+		// webhook doesn't support file uploads, so we add the url manually
+		if len(msg.Extra["file"]) > 0 {
+			for _, f := range msg.Extra["file"] {
+				fi := f.(config.FileInfo)
+				if fi.URL != "" {
+					msg.Text += fi.URL
+				}
+			}
+		}
+	}
+	iconURL := config.GetIconURL(msg, b.GetString("iconurl"))
+	matterMessage := matterhook.OMessage{
+		IconURL:  iconURL,
+		Channel:  msg.Channel,
+		UserName: msg.Username,
+		Text:     msg.Text,
+	}
+	if msg.Avatar != "" {
+		matterMessage.IconURL = msg.Avatar
+	}
+	err := b.mh.Send(matterMessage)
+	if err != nil {
+		b.Log.Info(err)
+		return err
+	}
+	return nil
+}

bridge/rocketchat/rocketchat.go

@@ -1,25 +1,53 @@
 package brocketchat
 
 import (
+	"errors"
+	"strings"
+	"sync"
+
 	"github.com/42wim/matterbridge/bridge"
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/42wim/matterbridge/bridge/helper"
 	"github.com/42wim/matterbridge/hook/rockethook"
 	"github.com/42wim/matterbridge/matterhook"
+	lru "github.com/hashicorp/golang-lru"
+	"github.com/matterbridge/Rocket.Chat.Go.SDK/models"
+	"github.com/matterbridge/Rocket.Chat.Go.SDK/realtime"
+	"github.com/matterbridge/Rocket.Chat.Go.SDK/rest"
 )
 
-type MMhook struct {
+type Brocketchat struct {
-	mh *matterhook.Client
+	mh    *matterhook.Client
-	rh *rockethook.Client
+	rh    *rockethook.Client
+	c     *realtime.Client
+	r     *rest.Client
+	cache *lru.Cache
+	*bridge.Config
+	messageChan chan models.Message
+	channelMap  map[string]string
+	user        *models.User
+	sync.RWMutex
 }
 
-type Brocketchat struct {
+const (
-	MMhook
+	sUserJoined       = "uj"
-	*bridge.Config
+	sUserLeft         = "ul"
-}
+	sRoomChangedTopic = "room_changed_topic"
+)
 
 func New(cfg *bridge.Config) bridge.Bridger {
-	return &Brocketchat{Config: cfg}
+	newCache, err := lru.New(100)
+	if err != nil {
+		cfg.Log.Fatalf("Could not create LRU cache for rocketchat bridge: %v", err)
+	}
+	b := &Brocketchat{
+		Config:      cfg,
+		messageChan: make(chan models.Message),
+		channelMap:  make(map[string]string),
+		cache:       newCache,
+	}
+	b.Log.Debugf("enabling rocketchat")
+	return b
 }
 
 func (b *Brocketchat) Command(cmd string) string {
@@ -27,69 +55,127 @@ func (b *Brocketchat) Command(cmd string) string {
 }
 
 func (b *Brocketchat) Connect() error {
-	b.Log.Info("Connecting webhooks")
+	if b.GetString("WebhookBindAddress") != "" {
-	b.mh = matterhook.New(b.GetString("WebhookURL"),
+		if err := b.doConnectWebhookBind(); err != nil {
-		matterhook.Config{InsecureSkipVerify: b.GetBool("SkipTLSVerify"),
+			return err
-			DisableServer: true})
+		}
-	b.rh = rockethook.New(b.GetString("WebhookURL"), rockethook.Config{BindAddress: b.GetString("WebhookBindAddress")})
+		go b.handleRocket()
-	go b.handleRocketHook()
+		return nil
+	}
+	switch {
+	case b.GetString("WebhookURL") != "":
+		if err := b.doConnectWebhookURL(); err != nil {
+			return err
+		}
+		go b.handleRocket()
+		return nil
+	case b.GetString("Login") != "":
+		b.Log.Info("Connecting using login/password (sending and receiving)")
+		err := b.apiLogin()
+		if err != nil {
+			return err
+		}
+		go b.handleRocket()
+	}
+	if b.GetString("WebhookBindAddress") == "" && b.GetString("WebhookURL") == "" &&
+		b.GetString("Login") == "" {
+		return errors.New("no connection method found. See that you have WebhookBindAddress, WebhookURL or Login/Password/Server configured")
+	}
 	return nil
 }
 
 func (b *Brocketchat) Disconnect() error {
 	return nil
-
 }
 
 func (b *Brocketchat) JoinChannel(channel config.ChannelInfo) error {
+	if b.c == nil {
+		return nil
+	}
+	id, err := b.c.GetChannelId(strings.TrimPrefix(channel.Name, "#"))
+	if err != nil {
+		return err
+	}
+	b.Lock()
+	b.channelMap[id] = channel.Name
+	b.Unlock()
+	mychannel := &models.Channel{ID: id, Name: strings.TrimPrefix(channel.Name, "#")}
+	if err := b.c.JoinChannel(id); err != nil {
+		return err
+	}
+	if err := b.c.SubscribeToMessageStream(mychannel, b.messageChan); err != nil {
+		return err
+	}
 	return nil
 }
 
 func (b *Brocketchat) Send(msg config.Message) (string, error) {
-	// ignore delete messages
+	// strip the # if people has set this
-	if msg.Event == config.EVENT_MSG_DELETE {
+	msg.Channel = strings.TrimPrefix(msg.Channel, "#")
-		return "", nil
+	channel := &models.Channel{ID: b.getChannelID(msg.Channel), Name: msg.Channel}
+
+	// Make a action /me of the message
+	if msg.Event == config.EventUserAction {
+		msg.Text = "_" + msg.Text + "_"
 	}
-	b.Log.Debugf("=> Receiving %#v", msg)
+
+	// Delete message
+	if msg.Event == config.EventMsgDelete {
+		if msg.ID == "" {
+			return "", nil
+		}
+		return msg.ID, b.c.DeleteMessage(&models.Message{ID: msg.ID})
+	}
+
+	// Use webhook to send the message
+	if b.GetString("WebhookURL") != "" {
+		return "", b.sendWebhook(&msg)
+	}
+
+	// Prepend nick if configured
+	if b.GetBool("PrefixMessagesWithNick") {
+		msg.Text = msg.Username + msg.Text
+	}
+
+	// Edit message if we have an ID
+	if msg.ID != "" {
+		return msg.ID, b.c.EditMessage(&models.Message{ID: msg.ID, Msg: msg.Text, RoomID: b.getChannelID(msg.Channel)})
+	}
+
+	// Upload a file if it exists
 	if msg.Extra != nil {
 		for _, rmsg := range helper.HandleExtra(&msg, b.General) {
-			iconURL := config.GetIconURL(&rmsg, b.GetString("iconurl"))
+			// strip the # if people has set this
-			matterMessage := matterhook.OMessage{IconURL: iconURL, Channel: rmsg.Channel, UserName: rmsg.Username, Text: rmsg.Text}
+			rmsg.Channel = strings.TrimPrefix(rmsg.Channel, "#")
-			b.mh.Send(matterMessage)
+			smsg := &models.Message{
-		}
+				RoomID: b.getChannelID(rmsg.Channel),
-		if len(msg.Extra["file"]) > 0 {
+				Msg:    rmsg.Username + rmsg.Text,
-			for _, f := range msg.Extra["file"] {
+				PostMessage: models.PostMessage{
-				fi := f.(config.FileInfo)
+					Avatar: rmsg.Avatar,
-				if fi.URL != "" {
+					Alias:  rmsg.Username,
-					msg.Text += fi.URL
+				},
-				}
+			}
+			if _, err := b.c.SendMessage(smsg); err != nil {
+				b.Log.Errorf("SendMessage failed: %s", err)
 			}
 		}
+		if len(msg.Extra["file"]) > 0 {
+			return "", b.handleUploadFile(&msg)
+		}
 	}
 
-	iconURL := config.GetIconURL(&msg, b.GetString("iconurl"))
+	smsg := &models.Message{
-	matterMessage := matterhook.OMessage{IconURL: iconURL}
+		RoomID: channel.ID,
-	matterMessage.Channel = msg.Channel
+		Msg:    msg.Text,
-	matterMessage.UserName = msg.Username
+		PostMessage: models.PostMessage{
-	matterMessage.Type = ""
+			Avatar: msg.Avatar,
-	matterMessage.Text = msg.Text
+			Alias:  msg.Username,
-	err := b.mh.Send(matterMessage)
+		},
-	if err != nil {
+	}
-		b.Log.Info(err)
+
+	rmsg, err := b.c.SendMessage(smsg)
+	if rmsg == nil {
 		return "", err
 	}
-	return "", nil
+	return rmsg.ID, err
-}
-
-func (b *Brocketchat) handleRocketHook() {
-	for {
-		message := b.rh.Receive()
-		b.Log.Debugf("Receiving from rockethook %#v", message)
-		// do not loop
-		if message.UserName == b.GetString("Nick") {
-			continue
-		}
-		b.Log.Debugf("<= Sending message from %s on %s to gateway", message.UserName, b.Account)
-		b.Remote <- config.Message{Text: message.Text, Username: message.UserName, Channel: message.ChannelName, Account: b.Account, UserID: message.UserID}
-	}
 }

bridge/slack/handlers.go

@@ -0,0 +1,373 @@
+package bslack
+
+import (
+	"errors"
+	"fmt"
+	"html"
+	"time"
+
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/42wim/matterbridge/bridge/helper"
+	"github.com/slack-go/slack"
+)
+
+// ErrEventIgnored is for events that should be ignored
+var ErrEventIgnored = errors.New("this event message should ignored")
+
+func (b *Bslack) handleSlack() {
+	messages := make(chan *config.Message)
+	if b.GetString(incomingWebhookConfig) != "" && b.GetString(tokenConfig) == "" {
+		b.Log.Debugf("Choosing webhooks based receiving")
+		go b.handleMatterHook(messages)
+	} else {
+		b.Log.Debugf("Choosing token based receiving")
+		go b.handleSlackClient(messages)
+	}
+	time.Sleep(time.Second)
+	b.Log.Debug("Start listening for Slack messages")
+	for message := range messages {
+		// don't do any action on deleted/typing messages
+		if message.Event != config.EventUserTyping && message.Event != config.EventMsgDelete {
+			b.Log.Debugf("<= Sending message from %s on %s to gateway", message.Username, b.Account)
+			// cleanup the message
+			message.Text = b.replaceMention(message.Text)
+			message.Text = b.replaceVariable(message.Text)
+			message.Text = b.replaceChannel(message.Text)
+			message.Text = b.replaceURL(message.Text)
+			message.Text = b.replaceb0rkedMarkDown(message.Text)
+			message.Text = html.UnescapeString(message.Text)
+
+			// Add the avatar
+			message.Avatar = b.users.getAvatar(message.UserID)
+		}
+
+		b.Log.Debugf("<= Message is %#v", message)
+		b.Remote <- *message
+	}
+}
+
+func (b *Bslack) handleSlackClient(messages chan *config.Message) {
+	for msg := range b.rtm.IncomingEvents {
+		if msg.Type != sUserTyping && msg.Type != sHello && msg.Type != sLatencyReport {
+			b.Log.Debugf("== Receiving event %#v", msg.Data)
+		}
+		switch ev := msg.Data.(type) {
+		case *slack.UserTypingEvent:
+			if !b.GetBool("ShowUserTyping") {
+				continue
+			}
+			rmsg, err := b.handleTypingEvent(ev)
+			if err == ErrEventIgnored {
+				continue
+			} else if err != nil {
+				b.Log.Errorf("%#v", err)
+				continue
+			}
+
+			messages <- rmsg
+		case *slack.MessageEvent:
+			if b.skipMessageEvent(ev) {
+				b.Log.Debugf("Skipped message: %#v", ev)
+				continue
+			}
+			rmsg, err := b.handleMessageEvent(ev)
+			if err != nil {
+				b.Log.Errorf("%#v", err)
+				continue
+			}
+			messages <- rmsg
+		case *slack.OutgoingErrorEvent:
+			b.Log.Debugf("%#v", ev.Error())
+		case *slack.ChannelJoinedEvent:
+			// When we join a channel we update the full list of users as
+			// well as the information for the channel that we joined as this
+			// should now tell that we are a member of it.
+			b.channels.registerChannel(ev.Channel)
+		case *slack.ConnectedEvent:
+			b.si = ev.Info
+			b.channels.populateChannels(true)
+			b.users.populateUsers(true)
+		case *slack.InvalidAuthEvent:
+			b.Log.Fatalf("Invalid Token %#v", ev)
+		case *slack.ConnectionErrorEvent:
+			b.Log.Errorf("Connection failed %#v %#v", ev.Error(), ev.ErrorObj)
+		case *slack.MemberJoinedChannelEvent:
+			b.users.populateUser(ev.User)
+		case *slack.HelloEvent, *slack.LatencyReport, *slack.ConnectingEvent:
+			continue
+		default:
+			b.Log.Debugf("Unhandled incoming event: %T", ev)
+		}
+	}
+}
+
+func (b *Bslack) handleMatterHook(messages chan *config.Message) {
+	for {
+		message := b.mh.Receive()
+		b.Log.Debugf("receiving from matterhook (slack) %#v", message)
+		if message.UserName == "slackbot" {
+			continue
+		}
+		messages <- &config.Message{
+			Username: message.UserName,
+			Text:     message.Text,
+			Channel:  message.ChannelName,
+		}
+	}
+}
+
+// skipMessageEvent skips event that need to be skipped :-)
+func (b *Bslack) skipMessageEvent(ev *slack.MessageEvent) bool {
+	switch ev.SubType {
+	case sChannelLeave, sChannelJoin:
+		return b.GetBool(noSendJoinConfig)
+	case sPinnedItem, sUnpinnedItem:
+		return true
+	case sChannelTopic, sChannelPurpose:
+		// Skip the event if our bot/user account changed the topic/purpose
+		if ev.User == b.si.User.ID {
+			return true
+		}
+	}
+
+	// Check for our callback ID
+	hasOurCallbackID := false
+	if len(ev.Blocks.BlockSet) == 1 {
+		block, ok := ev.Blocks.BlockSet[0].(*slack.SectionBlock)
+		hasOurCallbackID = ok && block.BlockID == "matterbridge_"+b.uuid
+	}
+
+	if ev.SubMessage != nil {
+		// It seems ev.SubMessage.Edited == nil when slack unfurls.
+		// Do not forward these messages. See Github issue #266.
+		if ev.SubMessage.ThreadTimestamp != ev.SubMessage.Timestamp &&
+			ev.SubMessage.Edited == nil {
+			return true
+		}
+		// see hidden subtypes at https://api.slack.com/events/message
+		// these messages are sent when we add a message to a thread #709
+		if ev.SubType == "message_replied" && ev.Hidden {
+			return true
+		}
+		if len(ev.SubMessage.Blocks.BlockSet) == 1 {
+			block, ok := ev.SubMessage.Blocks.BlockSet[0].(*slack.SectionBlock)
+			hasOurCallbackID = ok && block.BlockID == "matterbridge_"+b.uuid
+		}
+	}
+
+	// Skip any messages that we made ourselves or from 'slackbot' (see #527).
+	if ev.Username == sSlackBotUser ||
+		(b.rtm != nil && ev.Username == b.si.User.Name) || hasOurCallbackID {
+		return true
+	}
+
+	if len(ev.Files) > 0 {
+		return b.filesCached(ev.Files)
+	}
+	return false
+}
+
+func (b *Bslack) filesCached(files []slack.File) bool {
+	for i := range files {
+		if !b.fileCached(&files[i]) {
+			return false
+		}
+	}
+	return true
+}
+
+// handleMessageEvent handles the message events. Together with any called sub-methods,
+// this method implements the following event processing pipeline:
+//
+// 1. Check if the message should be ignored.
+//    NOTE: This is not actually part of the method below but is done just before it
+//          is called via the 'skipMessageEvent()' method.
+// 2. Populate the Matterbridge message that will be sent to the router based on the
+//    received event and logic that is common to all events that are not skipped.
+// 3. Detect and handle any message that is "status" related (think join channel, etc.).
+//    This might result in an early exit from the pipeline and passing of the
+//    pre-populated message to the Matterbridge router.
+// 4. Handle the specific case of messages that edit existing messages depending on
+//    configuration.
+// 5. Handle any attachments of the received event.
+// 6. Check that the Matterbridge message that we end up with after at the end of the
+//    pipeline is valid before sending it to the Matterbridge router.
+func (b *Bslack) handleMessageEvent(ev *slack.MessageEvent) (*config.Message, error) {
+	rmsg, err := b.populateReceivedMessage(ev)
+	if err != nil {
+		return nil, err
+	}
+
+	// Handle some message types early.
+	if b.handleStatusEvent(ev, rmsg) {
+		return rmsg, nil
+	}
+
+	b.handleAttachments(ev, rmsg)
+
+	// Verify that we have the right information and the message
+	// is well-formed before sending it out to the router.
+	if len(ev.Files) == 0 && (rmsg.Text == "" || rmsg.Username == "") {
+		if ev.BotID != "" {
+			// This is probably a webhook we couldn't resolve.
+			return nil, fmt.Errorf("message handling resulted in an empty bot message (probably an incoming webhook we couldn't resolve): %#v", ev)
+		}
+		if ev.SubMessage != nil {
+			return nil, fmt.Errorf("message handling resulted in an empty message: %#v with submessage %#v", ev, ev.SubMessage)
+		}
+		return nil, fmt.Errorf("message handling resulted in an empty message: %#v", ev)
+	}
+	return rmsg, nil
+}
+
+func (b *Bslack) handleStatusEvent(ev *slack.MessageEvent, rmsg *config.Message) bool {
+	switch ev.SubType {
+	case sChannelJoined, sMemberJoined:
+		// There's no further processing needed on channel events
+		// so we return 'true'.
+		return true
+	case sChannelJoin, sChannelLeave:
+		rmsg.Username = sSystemUser
+		rmsg.Event = config.EventJoinLeave
+	case sChannelTopic, sChannelPurpose:
+		b.channels.populateChannels(false)
+		rmsg.Event = config.EventTopicChange
+	case sMessageChanged:
+		rmsg.Text = ev.SubMessage.Text
+		// handle deleted thread starting messages
+		if ev.SubMessage.Text == "This message was deleted." {
+			rmsg.Event = config.EventMsgDelete
+			return true
+		}
+	case sMessageDeleted:
+		rmsg.Text = config.EventMsgDelete
+		rmsg.Event = config.EventMsgDelete
+		rmsg.ID = ev.DeletedTimestamp
+		// If a message is being deleted we do not need to process
+		// the event any further so we return 'true'.
+		return true
+	case sMeMessage:
+		rmsg.Event = config.EventUserAction
+	}
+	return false
+}
+
+func (b *Bslack) handleAttachments(ev *slack.MessageEvent, rmsg *config.Message) {
+	// File comments are set by the system (because there is no username given).
+	if ev.SubType == sFileComment {
+		rmsg.Username = sSystemUser
+	}
+
+	// See if we have some text in the attachments.
+	if rmsg.Text == "" {
+		for _, attach := range ev.Attachments {
+			if attach.Text != "" {
+				if attach.Title != "" {
+					rmsg.Text = attach.Title + "\n"
+				}
+				rmsg.Text += attach.Text
+			} else {
+				rmsg.Text = attach.Fallback
+			}
+		}
+	}
+
+	// Save the attachments, so that we can send them to other slack (compatible) bridges.
+	if len(ev.Attachments) > 0 {
+		rmsg.Extra[sSlackAttachment] = append(rmsg.Extra[sSlackAttachment], ev.Attachments)
+	}
+
+	// If we have files attached, download them (in memory) and put a pointer to it in msg.Extra.
+	for i := range ev.Files {
+		if err := b.handleDownloadFile(rmsg, &ev.Files[i], false); err != nil {
+			b.Log.Errorf("Could not download incoming file: %#v", err)
+		}
+	}
+}
+
+func (b *Bslack) handleTypingEvent(ev *slack.UserTypingEvent) (*config.Message, error) {
+	if ev.User == b.si.User.ID {
+		return nil, ErrEventIgnored
+	}
+	channelInfo, err := b.channels.getChannelByID(ev.Channel)
+	if err != nil {
+		return nil, err
+	}
+	return &config.Message{
+		Channel: channelInfo.Name,
+		Account: b.Account,
+		Event:   config.EventUserTyping,
+	}, nil
+}
+
+// handleDownloadFile handles file download
+func (b *Bslack) handleDownloadFile(rmsg *config.Message, file *slack.File, retry bool) error {
+	if b.fileCached(file) {
+		return nil
+	}
+	// Check that the file is neither too large nor blacklisted.
+	if err := helper.HandleDownloadSize(b.Log, rmsg, file.Name, int64(file.Size), b.General); err != nil {
+		b.Log.WithError(err).Infof("Skipping download of incoming file.")
+		return nil
+	}
+
+	// Actually download the file.
+	data, err := helper.DownloadFileAuth(file.URLPrivateDownload, "Bearer "+b.GetString(tokenConfig))
+	if err != nil {
+		return fmt.Errorf("download %s failed %#v", file.URLPrivateDownload, err)
+	}
+
+	if len(*data) != file.Size && !retry {
+		b.Log.Debugf("Data size (%d) is not equal to size declared (%d)\n", len(*data), file.Size)
+		time.Sleep(1 * time.Second)
+		return b.handleDownloadFile(rmsg, file, true)
+	}
+
+	// If a comment is attached to the file(s) it is in the 'Text' field of the Slack messge event
+	// and should be added as comment to only one of the files. We reset the 'Text' field to ensure
+	// that the comment is not duplicated.
+	comment := rmsg.Text
+	rmsg.Text = ""
+	helper.HandleDownloadData(b.Log, rmsg, file.Name, comment, file.URLPrivateDownload, data, b.General)
+	return nil
+}
+
+// handleGetChannelMembers handles messages containing the GetChannelMembers event
+// Sends a message to the router containing *config.ChannelMembers
+func (b *Bslack) handleGetChannelMembers(rmsg *config.Message) bool {
+	if rmsg.Event != config.EventGetChannelMembers {
+		return false
+	}
+
+	cMembers := b.channels.getChannelMembers(b.users)
+
+	extra := make(map[string][]interface{})
+	extra[config.EventGetChannelMembers] = append(extra[config.EventGetChannelMembers], cMembers)
+	msg := config.Message{
+		Extra:   extra,
+		Event:   config.EventGetChannelMembers,
+		Account: b.Account,
+	}
+
+	b.Log.Debugf("sending msg to remote %#v", msg)
+	b.Remote <- msg
+
+	return true
+}
+
+// fileCached implements Matterbridge's caching logic for files
+// shared via Slack.
+//
+// We consider that a file was cached if its ID was added in the last minute or
+// it's name was registered in the last 10 seconds. This ensures that an
+// identically named file but with different content will be uploaded correctly
+// (the assumption is that such name collisions will not occur within the given
+// timeframes).
+func (b *Bslack) fileCached(file *slack.File) bool {
+	if ts, ok := b.cache.Get("file" + file.ID); ok && time.Since(ts.(time.Time)) < time.Minute {
+		return true
+	} else if ts, ok = b.cache.Get("filename" + file.Name); ok && time.Since(ts.(time.Time)) < 10*time.Second {
+		return true
+	}
+	return false
+}

bridge/slack/helpers.go

@@ -0,0 +1,259 @@
+package bslack
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/sirupsen/logrus"
+	"github.com/slack-go/slack"
+)
+
+// populateReceivedMessage shapes the initial Matterbridge message that we will forward to the
+// router before we apply message-dependent modifications.
+func (b *Bslack) populateReceivedMessage(ev *slack.MessageEvent) (*config.Message, error) {
+	// Use our own func because rtm.GetChannelInfo doesn't work for private channels.
+	channel, err := b.channels.getChannelByID(ev.Channel)
+	if err != nil {
+		return nil, err
+	}
+
+	rmsg := &config.Message{
+		Text:     ev.Text,
+		Channel:  channel.Name,
+		Account:  b.Account,
+		ID:       ev.Timestamp,
+		Extra:    make(map[string][]interface{}),
+		ParentID: ev.ThreadTimestamp,
+		Protocol: b.Protocol,
+	}
+	if b.useChannelID {
+		rmsg.Channel = "ID:" + channel.ID
+	}
+
+	// Handle 'edit' messages.
+	if ev.SubMessage != nil && !b.GetBool(editDisableConfig) {
+		rmsg.ID = ev.SubMessage.Timestamp
+		if ev.SubMessage.ThreadTimestamp != ev.SubMessage.Timestamp {
+			b.Log.Debugf("SubMessage %#v", ev.SubMessage)
+			rmsg.Text = ev.SubMessage.Text + b.GetString(editSuffixConfig)
+		}
+	}
+
+	// For edits, only submessage has thread ts.
+	// Ensures edits to threaded messages maintain their prefix hint on the
+	// unthreaded end.
+	if ev.SubMessage != nil {
+		rmsg.ParentID = ev.SubMessage.ThreadTimestamp
+	}
+
+	if err = b.populateMessageWithUserInfo(ev, rmsg); err != nil {
+		return nil, err
+	}
+	return rmsg, err
+}
+
+func (b *Bslack) populateMessageWithUserInfo(ev *slack.MessageEvent, rmsg *config.Message) error {
+	if ev.SubType == sMessageDeleted || ev.SubType == sFileComment {
+		return nil
+	}
+
+	// First, deal with bot-originating messages but only do so when not using webhooks: we
+	// would not be able to distinguish which bot would be sending them.
+	if err := b.populateMessageWithBotInfo(ev, rmsg); err != nil {
+		return err
+	}
+
+	// Second, deal with "real" users if we have the necessary information.
+	var userID string
+	switch {
+	case ev.User != "":
+		userID = ev.User
+	case ev.SubMessage != nil && ev.SubMessage.User != "":
+		userID = ev.SubMessage.User
+	default:
+		return nil
+	}
+
+	user := b.users.getUser(userID)
+	if user == nil {
+		return fmt.Errorf("could not find information for user with id %s", ev.User)
+	}
+
+	rmsg.UserID = user.ID
+	rmsg.Username = user.Name
+	if user.Profile.DisplayName != "" {
+		rmsg.Username = user.Profile.DisplayName
+	}
+	return nil
+}
+
+func (b *Bslack) populateMessageWithBotInfo(ev *slack.MessageEvent, rmsg *config.Message) error {
+	if ev.BotID == "" || b.GetString(outgoingWebhookConfig) != "" {
+		return nil
+	}
+
+	var err error
+	var bot *slack.Bot
+	for {
+		bot, err = b.rtm.GetBotInfo(ev.BotID)
+		if err == nil {
+			break
+		}
+
+		if err = handleRateLimit(b.Log, err); err != nil {
+			b.Log.Errorf("Could not retrieve bot information: %#v", err)
+			return err
+		}
+	}
+	b.Log.Debugf("Found bot %#v", bot)
+
+	if bot.Name != "" {
+		rmsg.Username = bot.Name
+		if ev.Username != "" {
+			rmsg.Username = ev.Username
+		}
+		rmsg.UserID = bot.ID
+	}
+	return nil
+}
+
+var (
+	mentionRE        = regexp.MustCompile(`<@([a-zA-Z0-9]+)>`)
+	channelRE        = regexp.MustCompile(`<#[a-zA-Z0-9]+\|(.+?)>`)
+	variableRE       = regexp.MustCompile(`<!((?:subteam\^)?[a-zA-Z0-9]+)(?:\|@?(.+?))?>`)
+	urlRE            = regexp.MustCompile(`<(.*?)(\|.*?)?>`)
+	codeFenceRE      = regexp.MustCompile(`(?m)^` + "```" + `\w+$`)
+	topicOrPurposeRE = regexp.MustCompile(`(?s)(@.+) (cleared|set)(?: the)? channel (topic|purpose)(?:: (.*))?`)
+)
+
+func (b *Bslack) extractTopicOrPurpose(text string) (string, string) {
+	r := topicOrPurposeRE.FindStringSubmatch(text)
+	if len(r) == 5 {
+		action, updateType, extracted := r[2], r[3], r[4]
+		switch action {
+		case "set":
+			return updateType, extracted
+		case "cleared":
+			return updateType, ""
+		}
+	}
+	b.Log.Warnf("Encountered channel topic or purpose change message with unexpected format: %s", text)
+	return "unknown", ""
+}
+
+// @see https://api.slack.com/docs/message-formatting#linking_to_channels_and_users
+func (b *Bslack) replaceMention(text string) string {
+	replaceFunc := func(match string) string {
+		userID := strings.Trim(match, "@<>")
+		if username := b.users.getUsername(userID); userID != "" {
+			return "@" + username
+		}
+		return match
+	}
+	return mentionRE.ReplaceAllStringFunc(text, replaceFunc)
+}
+
+// @see https://api.slack.com/docs/message-formatting#linking_to_channels_and_users
+func (b *Bslack) replaceChannel(text string) string {
+	for _, r := range channelRE.FindAllStringSubmatch(text, -1) {
+		text = strings.Replace(text, r[0], "#"+r[1], 1)
+	}
+	return text
+}
+
+// @see https://api.slack.com/docs/message-formatting#variables
+func (b *Bslack) replaceVariable(text string) string {
+	for _, r := range variableRE.FindAllStringSubmatch(text, -1) {
+		if r[2] != "" {
+			text = strings.Replace(text, r[0], "@"+r[2], 1)
+		} else {
+			text = strings.Replace(text, r[0], "@"+r[1], 1)
+		}
+	}
+	return text
+}
+
+// @see https://api.slack.com/docs/message-formatting#linking_to_urls
+func (b *Bslack) replaceURL(text string) string {
+	for _, r := range urlRE.FindAllStringSubmatch(text, -1) {
+		if len(strings.TrimSpace(r[2])) == 1 { // A display text separator was found, but the text was blank
+			text = strings.Replace(text, r[0], "", 1)
+		} else {
+			text = strings.Replace(text, r[0], r[1], 1)
+		}
+	}
+	return text
+}
+
+func (b *Bslack) replaceb0rkedMarkDown(text string) string {
+	// taken from https://github.com/mattermost/mattermost-server/blob/master/app/slackimport.go
+	//
+	regexReplaceAllString := []struct {
+		regex *regexp.Regexp
+		rpl   string
+	}{
+		// bold
+		{
+			regexp.MustCompile(`(^|[\s.;,])\*(\S[^*\n]+)\*`),
+			"$1**$2**",
+		},
+		// strikethrough
+		{
+			regexp.MustCompile(`(^|[\s.;,])\~(\S[^~\n]+)\~`),
+			"$1~~$2~~",
+		},
+		// single paragraph blockquote
+		// Slack converts > character to &gt;
+		{
+			regexp.MustCompile(`(?sm)^&gt;`),
+			">",
+		},
+	}
+	for _, rule := range regexReplaceAllString {
+		text = rule.regex.ReplaceAllString(text, rule.rpl)
+	}
+	return text
+}
+
+func (b *Bslack) replaceCodeFence(text string) string {
+	return codeFenceRE.ReplaceAllString(text, "```")
+}
+
+// getUsersInConversation returns an array of userIDs that are members of channelID
+func (b *Bslack) getUsersInConversation(channelID string) ([]string, error) {
+	channelMembers := []string{}
+	for {
+		queryParams := &slack.GetUsersInConversationParameters{
+			ChannelID: channelID,
+		}
+
+		members, nextCursor, err := b.sc.GetUsersInConversation(queryParams)
+		if err != nil {
+			if err = handleRateLimit(b.Log, err); err != nil {
+				return channelMembers, fmt.Errorf("Could not retrieve users in channels: %#v", err)
+			}
+			continue
+		}
+
+		channelMembers = append(channelMembers, members...)
+
+		if nextCursor == "" {
+			break
+		}
+		queryParams.Cursor = nextCursor
+	}
+	return channelMembers, nil
+}
+
+func handleRateLimit(log *logrus.Entry, err error) error {
+	rateLimit, ok := err.(*slack.RateLimitedError)
+	if !ok {
+		return err
+	}
+	log.Infof("Rate-limited by Slack. Sleeping for %v", rateLimit.RetryAfter)
+	time.Sleep(rateLimit.RetryAfter)
+	return nil
+}

bridge/slack/helpers_test.go

@@ -0,0 +1,36 @@
+package bslack
+
+import (
+	"io/ioutil"
+	"testing"
+
+	"github.com/42wim/matterbridge/bridge"
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestExtractTopicOrPurpose(t *testing.T) {
+	testcases := map[string]struct {
+		input          string
+		wantChangeType string
+		wantOutput     string
+	}{
+		"success - topic type":   {"@someone set channel topic: foo bar", "topic", "foo bar"},
+		"success - purpose type": {"@someone set channel purpose: foo bar", "purpose", "foo bar"},
+		"success - one line":     {"@someone set channel topic: foo bar", "topic", "foo bar"},
+		"success - multi-line":   {"@someone set channel topic: foo\nbar", "topic", "foo\nbar"},
+		"success - cleared":      {"@someone cleared channel topic", "topic", ""},
+		"error - unhandled":      {"some unmatched message", "unknown", ""},
+	}
+
+	logger := logrus.New()
+	logger.SetOutput(ioutil.Discard)
+	cfg := &bridge.Config{Bridge: &bridge.Bridge{Log: logrus.NewEntry(logger)}}
+	b := newBridge(cfg)
+	for name, tc := range testcases {
+		gotChangeType, gotOutput := b.extractTopicOrPurpose(tc.input)
+
+		assert.Equalf(t, tc.wantChangeType, gotChangeType, "This testcase failed: %s", name)
+		assert.Equalf(t, tc.wantOutput, gotOutput, "This testcase failed: %s", name)
+	}
+}

bridge/slack/legacy.go

@@ -0,0 +1,80 @@
+package bslack
+
+import (
+	"errors"
+
+	"github.com/42wim/matterbridge/bridge"
+	"github.com/42wim/matterbridge/matterhook"
+	"github.com/slack-go/slack"
+)
+
+type BLegacy struct {
+	*Bslack
+}
+
+func NewLegacy(cfg *bridge.Config) bridge.Bridger {
+	b := &BLegacy{Bslack: newBridge(cfg)}
+	b.legacy = true
+	return b
+}
+
+func (b *BLegacy) Connect() error {
+	b.RLock()
+	defer b.RUnlock()
+	if b.GetString(incomingWebhookConfig) != "" {
+		switch {
+		case b.GetString(outgoingWebhookConfig) != "":
+			b.Log.Info("Connecting using webhookurl (sending) and webhookbindaddress (receiving)")
+			b.mh = matterhook.New(b.GetString(outgoingWebhookConfig), matterhook.Config{
+				InsecureSkipVerify: b.GetBool(skipTLSConfig),
+				BindAddress:        b.GetString(incomingWebhookConfig),
+			})
+		case b.GetString(tokenConfig) != "":
+			b.Log.Info("Connecting using token (sending)")
+			b.sc = slack.New(b.GetString(tokenConfig))
+			b.rtm = b.sc.NewRTM()
+			go b.rtm.ManageConnection()
+			b.Log.Info("Connecting using webhookbindaddress (receiving)")
+			b.mh = matterhook.New(b.GetString(outgoingWebhookConfig), matterhook.Config{
+				InsecureSkipVerify: b.GetBool(skipTLSConfig),
+				BindAddress:        b.GetString(incomingWebhookConfig),
+			})
+		default:
+			b.Log.Info("Connecting using webhookbindaddress (receiving)")
+			b.mh = matterhook.New(b.GetString(outgoingWebhookConfig), matterhook.Config{
+				InsecureSkipVerify: b.GetBool(skipTLSConfig),
+				BindAddress:        b.GetString(incomingWebhookConfig),
+			})
+		}
+		go b.handleSlack()
+		return nil
+	}
+	if b.GetString(outgoingWebhookConfig) != "" {
+		b.Log.Info("Connecting using webhookurl (sending)")
+		b.mh = matterhook.New(b.GetString(outgoingWebhookConfig), matterhook.Config{
+			InsecureSkipVerify: b.GetBool(skipTLSConfig),
+			DisableServer:      true,
+		})
+		if b.GetString(tokenConfig) != "" {
+			b.Log.Info("Connecting using token (receiving)")
+			b.sc = slack.New(b.GetString(tokenConfig), slack.OptionDebug(b.GetBool("debug")))
+			b.channels = newChannelManager(b.Log, b.sc)
+			b.users = newUserManager(b.Log, b.sc)
+			b.rtm = b.sc.NewRTM()
+			go b.rtm.ManageConnection()
+			go b.handleSlack()
+		}
+	} else if b.GetString(tokenConfig) != "" {
+		b.Log.Info("Connecting using token (sending and receiving)")
+		b.sc = slack.New(b.GetString(tokenConfig), slack.OptionDebug(b.GetBool("debug")))
+		b.channels = newChannelManager(b.Log, b.sc)
+		b.users = newUserManager(b.Log, b.sc)
+		b.rtm = b.sc.NewRTM()
+		go b.rtm.ManageConnection()
+		go b.handleSlack()
+	}
+	if b.GetString(incomingWebhookConfig) == "" && b.GetString(outgoingWebhookConfig) == "" && b.GetString(tokenConfig) == "" {
+		return errors.New("no connection method found. See that you have WebhookBindAddress, WebhookURL or Token configured")
+	}
+	return nil
+}

bridge/slack/users_channels.go

@@ -0,0 +1,336 @@
+package bslack
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/sirupsen/logrus"
+	"github.com/slack-go/slack"
+)
+
+const minimumRefreshInterval = 10 * time.Second
+
+type users struct {
+	log *logrus.Entry
+	sc  *slack.Client
+
+	users           map[string]*slack.User
+	usersMutex      sync.RWMutex
+	usersSyncPoints map[string]chan struct{}
+
+	refreshInProgress bool
+	earliestRefresh   time.Time
+	refreshMutex      sync.Mutex
+}
+
+func newUserManager(log *logrus.Entry, sc *slack.Client) *users {
+	return &users{
+		log:             log,
+		sc:              sc,
+		users:           make(map[string]*slack.User),
+		usersSyncPoints: make(map[string]chan struct{}),
+		earliestRefresh: time.Now(),
+	}
+}
+
+func (b *users) getUser(id string) *slack.User {
+	b.usersMutex.RLock()
+	user, ok := b.users[id]
+	b.usersMutex.RUnlock()
+	if ok {
+		return user
+	}
+	b.populateUser(id)
+	b.usersMutex.RLock()
+	defer b.usersMutex.RUnlock()
+
+	return b.users[id]
+}
+
+func (b *users) getUsername(id string) string {
+	if user := b.getUser(id); user != nil {
+		if user.Profile.DisplayName != "" {
+			return user.Profile.DisplayName
+		}
+		return user.Name
+	}
+	b.log.Warnf("Could not find user with ID '%s'", id)
+	return ""
+}
+
+func (b *users) getAvatar(id string) string {
+	if user := b.getUser(id); user != nil {
+		return user.Profile.Image48
+	}
+	return ""
+}
+
+func (b *users) populateUser(userID string) {
+	for {
+		b.usersMutex.Lock()
+		_, exists := b.users[userID]
+		if exists {
+			// already in cache
+			b.usersMutex.Unlock()
+			return
+		}
+
+		if syncPoint, ok := b.usersSyncPoints[userID]; ok {
+			// Another goroutine is already populating this user for us so wait on it to finish.
+			b.usersMutex.Unlock()
+			<-syncPoint
+			// We do not return and iterate again to check that the entry does indeed exist
+			// in case the previous query failed for some reason.
+		} else {
+			b.usersSyncPoints[userID] = make(chan struct{})
+			defer func() {
+				// Wake up any waiting goroutines and remove the synchronization point.
+				close(b.usersSyncPoints[userID])
+				delete(b.usersSyncPoints, userID)
+			}()
+			break
+		}
+	}
+
+	// Do not hold the lock while fetching information from Slack
+	// as this might take an unbounded amount of time.
+	b.usersMutex.Unlock()
+
+	user, err := b.sc.GetUserInfo(userID)
+	if err != nil {
+		b.log.Debugf("GetUserInfo failed for %v: %v", userID, err)
+		return
+	}
+
+	b.usersMutex.Lock()
+	defer b.usersMutex.Unlock()
+
+	// Register user information.
+	b.users[userID] = user
+}
+
+func (b *users) populateUsers(wait bool) {
+	b.refreshMutex.Lock()
+	if !wait && (time.Now().Before(b.earliestRefresh) || b.refreshInProgress) {
+		b.log.Debugf("Not refreshing user list as it was done less than %v ago.", minimumRefreshInterval)
+		b.refreshMutex.Unlock()
+
+		return
+	}
+	for b.refreshInProgress {
+		b.refreshMutex.Unlock()
+		time.Sleep(time.Second)
+		b.refreshMutex.Lock()
+	}
+	b.refreshInProgress = true
+	b.refreshMutex.Unlock()
+
+	newUsers := map[string]*slack.User{}
+	pagination := b.sc.GetUsersPaginated(slack.GetUsersOptionLimit(200))
+	count := 0
+	for {
+		var err error
+		pagination, err = pagination.Next(context.Background())
+		time.Sleep(time.Second)
+		if err != nil {
+			if pagination.Done(err) {
+				break
+			}
+
+			if err = handleRateLimit(b.log, err); err != nil {
+				b.log.Errorf("Could not retrieve users: %#v", err)
+				return
+			}
+			continue
+		}
+
+		for i := range pagination.Users {
+			newUsers[pagination.Users[i].ID] = &pagination.Users[i]
+		}
+		b.log.Debugf("getting %d users", len(pagination.Users))
+		count++
+		// more > 2000 users, slack will complain and ratelimit. break
+		if count > 10 {
+			b.log.Info("Large slack detected > 2000 users, skipping loading complete userlist.")
+			break
+		}
+	}
+
+	b.usersMutex.Lock()
+	defer b.usersMutex.Unlock()
+	b.users = newUsers
+
+	b.refreshMutex.Lock()
+	defer b.refreshMutex.Unlock()
+	b.earliestRefresh = time.Now().Add(minimumRefreshInterval)
+	b.refreshInProgress = false
+}
+
+type channels struct {
+	log *logrus.Entry
+	sc  *slack.Client
+
+	channelsByID   map[string]*slack.Channel
+	channelsByName map[string]*slack.Channel
+	channelsMutex  sync.RWMutex
+
+	channelMembers      map[string][]string
+	channelMembersMutex sync.RWMutex
+
+	refreshInProgress bool
+	earliestRefresh   time.Time
+	refreshMutex      sync.Mutex
+}
+
+func newChannelManager(log *logrus.Entry, sc *slack.Client) *channels {
+	return &channels{
+		log:             log,
+		sc:              sc,
+		channelsByID:    make(map[string]*slack.Channel),
+		channelsByName:  make(map[string]*slack.Channel),
+		earliestRefresh: time.Now(),
+	}
+}
+
+func (b *channels) getChannel(channel string) (*slack.Channel, error) {
+	if strings.HasPrefix(channel, "ID:") {
+		return b.getChannelByID(strings.TrimPrefix(channel, "ID:"))
+	}
+	return b.getChannelByName(channel)
+}
+
+func (b *channels) getChannelByName(name string) (*slack.Channel, error) {
+	return b.getChannelBy(name, b.channelsByName)
+}
+
+func (b *channels) getChannelByID(id string) (*slack.Channel, error) {
+	return b.getChannelBy(id, b.channelsByID)
+}
+
+func (b *channels) getChannelBy(lookupKey string, lookupMap map[string]*slack.Channel) (*slack.Channel, error) {
+	b.channelsMutex.RLock()
+	defer b.channelsMutex.RUnlock()
+
+	if channel, ok := lookupMap[lookupKey]; ok {
+		return channel, nil
+	}
+	return nil, fmt.Errorf("channel %s not found", lookupKey)
+}
+
+func (b *channels) getChannelMembers(users *users) config.ChannelMembers {
+	b.channelMembersMutex.RLock()
+	defer b.channelMembersMutex.RUnlock()
+
+	membersInfo := config.ChannelMembers{}
+	for channelID, members := range b.channelMembers {
+		for _, member := range members {
+			channelName := ""
+			userName := ""
+			userNick := ""
+			user := users.getUser(member)
+			if user != nil {
+				userName = user.Name
+				userNick = user.Profile.DisplayName
+			}
+			channel, _ := b.getChannelByID(channelID)
+			if channel != nil {
+				channelName = channel.Name
+			}
+			memberInfo := config.ChannelMember{
+				Username:    userName,
+				Nick:        userNick,
+				UserID:      member,
+				ChannelID:   channelID,
+				ChannelName: channelName,
+			}
+			membersInfo = append(membersInfo, memberInfo)
+		}
+	}
+	return membersInfo
+}
+
+func (b *channels) registerChannel(channel slack.Channel) {
+	b.channelsMutex.Lock()
+	defer b.channelsMutex.Unlock()
+
+	b.channelsByID[channel.ID] = &channel
+	b.channelsByName[channel.Name] = &channel
+}
+
+func (b *channels) populateChannels(wait bool) {
+	b.refreshMutex.Lock()
+	if !wait && (time.Now().Before(b.earliestRefresh) || b.refreshInProgress) {
+		b.log.Debugf("Not refreshing channel list as it was done less than %v seconds ago.", minimumRefreshInterval)
+		b.refreshMutex.Unlock()
+		return
+	}
+	for b.refreshInProgress {
+		b.refreshMutex.Unlock()
+		time.Sleep(time.Second)
+		b.refreshMutex.Lock()
+	}
+	b.refreshInProgress = true
+	b.refreshMutex.Unlock()
+
+	newChannelsByID := map[string]*slack.Channel{}
+	newChannelsByName := map[string]*slack.Channel{}
+	newChannelMembers := make(map[string][]string)
+
+	// We only retrieve public and private channels, not IMs
+	// and MPIMs as those do not have a channel name.
+	queryParams := &slack.GetConversationsParameters{
+		ExcludeArchived: "true",
+		Types:           []string{"public_channel,private_channel"},
+	}
+	for {
+		channels, nextCursor, err := b.sc.GetConversations(queryParams)
+		if err != nil {
+			if err = handleRateLimit(b.log, err); err != nil {
+				b.log.Errorf("Could not retrieve channels: %#v", err)
+				return
+			}
+			continue
+		}
+
+		for i := range channels {
+			newChannelsByID[channels[i].ID] = &channels[i]
+			newChannelsByName[channels[i].Name] = &channels[i]
+			// also find all the members in every channel
+			// comment for now, issues on big slacks
+			/*
+				members, err := b.getUsersInConversation(channels[i].ID)
+				if err != nil {
+					if err = b.handleRateLimit(err); err != nil {
+						b.Log.Errorf("Could not retrieve channel members: %#v", err)
+						return
+					}
+					continue
+				}
+				newChannelMembers[channels[i].ID] = members
+			*/
+		}
+
+		if nextCursor == "" {
+			break
+		}
+		queryParams.Cursor = nextCursor
+	}
+
+	b.channelsMutex.Lock()
+	defer b.channelsMutex.Unlock()
+	b.channelsByID = newChannelsByID
+	b.channelsByName = newChannelsByName
+
+	b.channelMembersMutex.Lock()
+	defer b.channelMembersMutex.Unlock()
+	b.channelMembers = newChannelMembers
+
+	b.refreshMutex.Lock()
+	defer b.refreshMutex.Unlock()
+	b.earliestRefresh = time.Now().Add(minimumRefreshInterval)
+	b.refreshInProgress = false
+}

bridge/sshchat/sshchat.go

@@ -9,7 +9,6 @@ import (
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/42wim/matterbridge/bridge/helper"
 	"github.com/shazow/ssh-chat/sshd"
-	log "github.com/sirupsen/logrus"
 )
 
 type Bsshchat struct {
@@ -23,21 +22,35 @@ func New(cfg *bridge.Config) bridge.Bridger {
 }
 
 func (b *Bsshchat) Connect() error {
-	var err error
 	b.Log.Infof("Connecting %s", b.GetString("Server"))
+
+	// connHandler will be called by 'sshd.ConnectShell()' below
+	// once the connection is established in order to handle it.
+	connErr := make(chan error, 1) // Needs to be buffered.
+	connSignal := make(chan struct{})
+	connHandler := func(r io.Reader, w io.WriteCloser) error {
+		b.r = bufio.NewScanner(r)
+		b.r.Scan()
+		b.w = w
+		if _, err := b.w.Write([]byte("/theme mono\r\n/quiet\r\n")); err != nil {
+			return err
+		}
+		close(connSignal) // Connection is established so we can signal the success.
+		return b.handleSSHChat()
+	}
+
 	go func() {
-		err = sshd.ConnectShell(b.GetString("Server"), b.GetString("Nick"), func(r io.Reader, w io.WriteCloser) error {
+		// As a successful connection will result in this returning after the Connection
-			b.r = bufio.NewScanner(r)
+		// method has already returned point we NEED to have a buffered channel to still
-			b.w = w
+		// be able to write.
-			b.r.Scan()
+		connErr <- sshd.ConnectShell(b.GetString("Server"), b.GetString("Nick"), connHandler)
-			w.Write([]byte("/theme mono\r\n"))
-			b.handleSshChat()
-			return nil
-		})
 	}()
-	if err != nil {
+
-		b.Log.Debugf("%#v", err)
+	select {
+	case err := <-connErr:
+		b.Log.Error("Connection failed")
 		return err
+	case <-connSignal:
 	}
 	b.Log.Info("Connection succeeded")
 	return nil
@@ -53,33 +66,22 @@ func (b *Bsshchat) JoinChannel(channel config.ChannelInfo) error {
 
 func (b *Bsshchat) Send(msg config.Message) (string, error) {
 	// ignore delete messages
-	if msg.Event == config.EVENT_MSG_DELETE {
+	if msg.Event == config.EventMsgDelete {
 		return "", nil
 	}
 	b.Log.Debugf("=> Receiving %#v", msg)
 	if msg.Extra != nil {
 		for _, rmsg := range helper.HandleExtra(&msg, b.General) {
-			b.w.Write([]byte(rmsg.Username + rmsg.Text + "\r\n"))
+			if _, err := b.w.Write([]byte(rmsg.Username + rmsg.Text + "\r\n")); err != nil {
+				b.Log.Errorf("Could not send extra message: %#v", err)
+			}
 		}
 		if len(msg.Extra["file"]) > 0 {
-			for _, f := range msg.Extra["file"] {
+			return b.handleUploadFile(&msg)
-				fi := f.(config.FileInfo)
-				if fi.Comment != "" {
-					msg.Text += fi.Comment + ": "
-				}
-				if fi.URL != "" {
-					msg.Text = fi.URL
-					if fi.Comment != "" {
-						msg.Text = fi.Comment + ": " + fi.URL
-					}
-				}
-				b.w.Write([]byte(msg.Username + msg.Text))
-			}
-			return "", nil
 		}
 	}
-	b.w.Write([]byte(msg.Username + msg.Text + "\r\n"))
+	_, err := b.w.Write([]byte(msg.Username + msg.Text + "\r\n"))
-	return "", nil
+	return "", err
 }
 
 /*
@@ -113,7 +115,7 @@ func stripPrompt(s string) string {
 	return s[pos+3:]
 }
 
-func (b *Bsshchat) handleSshChat() error {
+func (b *Bsshchat) handleSSHChat() error {
 	/*
 		done := b.sshchatKeepAlive()
 		defer close(done)
@@ -125,17 +127,43 @@ func (b *Bsshchat) handleSshChat() error {
 			if !strings.Contains(b.r.Text(), "\033[K") {
 				continue
 			}
+			if strings.Contains(b.r.Text(), "Rate limiting is in effect") {
+				continue
+			}
+			// skip our own messages
+			if !strings.HasPrefix(b.r.Text(), "["+b.GetString("Nick")+"] \x1b") {
+				continue
+			}
 			res := strings.Split(stripPrompt(b.r.Text()), ":")
 			if res[0] == "-> Set theme" {
 				wait = false
-				log.Debugf("mono found, allowing")
+				b.Log.Debugf("mono found, allowing")
 				continue
 			}
 			if !wait {
 				b.Log.Debugf("<= Message %#v", res)
-				rmsg := config.Message{Username: res[0], Text: strings.Join(res[1:], ":"), Channel: "sshchat", Account: b.Account, UserID: "nick"}
+				rmsg := config.Message{Username: res[0], Text: strings.TrimSpace(strings.Join(res[1:], ":")), Channel: "sshchat", Account: b.Account, UserID: "nick"}
 				b.Remote <- rmsg
 			}
 		}
 	}
 }
+
+func (b *Bsshchat) handleUploadFile(msg *config.Message) (string, error) {
+	for _, f := range msg.Extra["file"] {
+		fi := f.(config.FileInfo)
+		if fi.Comment != "" {
+			msg.Text += fi.Comment + ": "
+		}
+		if fi.URL != "" {
+			msg.Text = fi.URL
+			if fi.Comment != "" {
+				msg.Text = fi.Comment + ": " + fi.URL
+			}
+		}
+		if _, err := b.w.Write([]byte(msg.Username + msg.Text + "\r\n")); err != nil {
+			b.Log.Errorf("Could not send file message: %#v", err)
+		}
+	}
+	return "", nil
+}

bridge/steam/handlers.go

@@ -0,0 +1,126 @@
+package bsteam
+
+import (
+	"fmt"
+	"strconv"
+
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/Philipp15b/go-steam"
+	"github.com/Philipp15b/go-steam/protocol/steamlang"
+)
+
+func (b *Bsteam) handleChatMsg(e *steam.ChatMsgEvent) {
+	b.Log.Debugf("Receiving ChatMsgEvent: %#v", e)
+	b.Log.Debugf("<= Sending message from %s on %s to gateway", b.getNick(e.ChatterId), b.Account)
+	var channel int64
+	if e.ChatRoomId == 0 {
+		channel = int64(e.ChatterId)
+	} else {
+		// for some reason we have to remove 0x18000000000000
+		// TODO
+		// https://github.com/42wim/matterbridge/pull/630#discussion_r238102751
+		// channel = int64(e.ChatRoomId) & 0xfffffffffffff
+		channel = int64(e.ChatRoomId) - 0x18000000000000
+	}
+	msg := config.Message{
+		Username: b.getNick(e.ChatterId),
+		Text:     e.Message,
+		Channel:  strconv.FormatInt(channel, 10),
+		Account:  b.Account,
+		UserID:   strconv.FormatInt(int64(e.ChatterId), 10),
+	}
+	b.Remote <- msg
+}
+
+func (b *Bsteam) handleEvents() {
+	myLoginInfo := &steam.LogOnDetails{
+		Username: b.GetString("Login"),
+		Password: b.GetString("Password"),
+		AuthCode: b.GetString("AuthCode"),
+	}
+	// TODO Attempt to read existing auth hash to avoid steam guard.
+	// Maybe works
+	//myLoginInfo.SentryFileHash, _ = ioutil.ReadFile("sentry")
+	for event := range b.c.Events() {
+		switch e := event.(type) {
+		case *steam.ChatMsgEvent:
+			b.handleChatMsg(e)
+		case *steam.PersonaStateEvent:
+			b.Log.Debugf("PersonaStateEvent: %#v\n", e)
+			b.Lock()
+			b.userMap[e.FriendId] = e.Name
+			b.Unlock()
+		case *steam.ConnectedEvent:
+			b.c.Auth.LogOn(myLoginInfo)
+		case *steam.MachineAuthUpdateEvent:
+		// TODO sentry files for 2 auth
+		/*
+			b.Log.Info("authupdate", e)
+			b.Log.Info("hash", e.Hash)
+			ioutil.WriteFile("sentry", e.Hash, 0666)
+		*/
+		case *steam.LogOnFailedEvent:
+			b.Log.Info("Logon failed", e)
+			err := b.handleLogOnFailed(e, myLoginInfo)
+			if err != nil {
+				b.Log.Error(err)
+				return
+			}
+		case *steam.LoggedOnEvent:
+			b.Log.Debugf("LoggedOnEvent: %#v", e)
+			b.connected <- struct{}{}
+			b.Log.Debugf("setting online")
+			b.c.Social.SetPersonaState(steamlang.EPersonaState_Online)
+		case *steam.DisconnectedEvent:
+			b.Log.Info("Disconnected")
+			b.Log.Info("Attempting to reconnect...")
+			b.c.Connect()
+		case steam.FatalErrorEvent:
+			b.Log.Errorf("steam FatalErrorEvent: %#v", e)
+		default:
+			b.Log.Debugf("unknown event %#v", e)
+		}
+	}
+}
+
+func (b *Bsteam) handleLogOnFailed(e *steam.LogOnFailedEvent, myLoginInfo *steam.LogOnDetails) error {
+	switch e.Result {
+	case steamlang.EResult_AccountLoginDeniedNeedTwoFactor:
+		b.Log.Info("Steam guard isn't letting me in! Enter 2FA code:")
+		var code string
+		fmt.Scanf("%s", &code)
+		// TODO https://github.com/42wim/matterbridge/pull/630#discussion_r238103978
+		myLoginInfo.TwoFactorCode = code
+	case steamlang.EResult_AccountLogonDenied:
+		b.Log.Info("Steam guard isn't letting me in! Enter auth code:")
+		var code string
+		fmt.Scanf("%s", &code)
+		// TODO https://github.com/42wim/matterbridge/pull/630#discussion_r238103978
+		myLoginInfo.AuthCode = code
+	case steamlang.EResult_InvalidLoginAuthCode:
+		return fmt.Errorf("Steam guard: invalid login auth code: %#v ", e.Result)
+	default:
+		return fmt.Errorf("LogOnFailedEvent: %#v ", e.Result)
+		// TODO: Handle EResult_InvalidLoginAuthCode
+	}
+	return nil
+}
+
+// handleFileInfo handles config.FileInfo and adds correct file comment or URL to msg.Text.
+// Returns error if cast fails.
+func (b *Bsteam) handleFileInfo(msg *config.Message, f interface{}) error {
+	if _, ok := f.(config.FileInfo); !ok {
+		return fmt.Errorf("handleFileInfo cast failed %#v", f)
+	}
+	fi := f.(config.FileInfo)
+	if fi.Comment != "" {
+		msg.Text += fi.Comment + ": "
+	}
+	if fi.URL != "" {
+		msg.Text = fi.URL
+		if fi.Comment != "" {
+			msg.Text = fi.Comment + ": " + fi.URL
+		}
+	}
+	return nil
+}

bridge/steam/steam.go

@@ -2,6 +2,8 @@ package bsteam
 
 import (
 	"fmt"
+	"sync"
+	"time"
 
 	"github.com/42wim/matterbridge/bridge"
 	"github.com/42wim/matterbridge/bridge/config"
@@ -9,10 +11,6 @@ import (
 	"github.com/Philipp15b/go-steam"
 	"github.com/Philipp15b/go-steam/protocol/steamlang"
 	"github.com/Philipp15b/go-steam/steamid"
-	//"io/ioutil"
-	"strconv"
-	"sync"
-	"time"
 )
 
 type Bsteam struct {
@@ -61,7 +59,7 @@ func (b *Bsteam) JoinChannel(channel config.ChannelInfo) error {
 
 func (b *Bsteam) Send(msg config.Message) (string, error) {
 	// ignore delete messages
-	if msg.Event == config.EVENT_MSG_DELETE {
+	if msg.Event == config.EventMsgDelete {
 		return "", nil
 	}
 	id, err := steamid.NewId(msg.Channel)
@@ -74,22 +72,13 @@ func (b *Bsteam) Send(msg config.Message) (string, error) {
 		for _, rmsg := range helper.HandleExtra(&msg, b.General) {
 			b.c.Social.SendMessage(id, steamlang.EChatEntryType_ChatMsg, rmsg.Username+rmsg.Text)
 		}
-		if len(msg.Extra["file"]) > 0 {
+		for i := range msg.Extra["file"] {
-			for _, f := range msg.Extra["file"] {
+			if err := b.handleFileInfo(&msg, msg.Extra["file"][i]); err != nil {
-				fi := f.(config.FileInfo)
+				b.Log.Error(err)
-				if fi.Comment != "" {
-					msg.Text += fi.Comment + ": "
-				}
-				if fi.URL != "" {
-					msg.Text = fi.URL
-					if fi.Comment != "" {
-						msg.Text = fi.Comment + ": " + fi.URL
-					}
-				}
-				b.c.Social.SendMessage(id, steamlang.EChatEntryType_ChatMsg, msg.Username+msg.Text)
 			}
-			return "", nil
+			b.c.Social.SendMessage(id, steamlang.EChatEntryType_ChatMsg, msg.Username+msg.Text)
 		}
+		return "", nil
 	}
 
 	b.c.Social.SendMessage(id, steamlang.EChatEntryType_ChatMsg, msg.Username+msg.Text)
@@ -104,80 +93,3 @@ func (b *Bsteam) getNick(id steamid.SteamId) string {
 	}
 	return "unknown"
 }
-
-func (b *Bsteam) handleEvents() {
-	myLoginInfo := new(steam.LogOnDetails)
-	myLoginInfo.Username = b.GetString("Login")
-	myLoginInfo.Password = b.GetString("Password")
-	myLoginInfo.AuthCode = b.GetString("AuthCode")
-	// Attempt to read existing auth hash to avoid steam guard.
-	// Maybe works
-	//myLoginInfo.SentryFileHash, _ = ioutil.ReadFile("sentry")
-	for event := range b.c.Events() {
-		//b.Log.Info(event)
-		switch e := event.(type) {
-		case *steam.ChatMsgEvent:
-			b.Log.Debugf("Receiving ChatMsgEvent: %#v", e)
-			b.Log.Debugf("<= Sending message from %s on %s to gateway", b.getNick(e.ChatterId), b.Account)
-			var channel int64
-			if e.ChatRoomId == 0 {
-				channel = int64(e.ChatterId)
-			} else {
-				// for some reason we have to remove 0x18000000000000
-				channel = int64(e.ChatRoomId) - 0x18000000000000
-			}
-			msg := config.Message{Username: b.getNick(e.ChatterId), Text: e.Message, Channel: strconv.FormatInt(channel, 10), Account: b.Account, UserID: strconv.FormatInt(int64(e.ChatterId), 10)}
-			b.Remote <- msg
-		case *steam.PersonaStateEvent:
-			b.Log.Debugf("PersonaStateEvent: %#v\n", e)
-			b.Lock()
-			b.userMap[e.FriendId] = e.Name
-			b.Unlock()
-		case *steam.ConnectedEvent:
-			b.c.Auth.LogOn(myLoginInfo)
-		case *steam.MachineAuthUpdateEvent:
-			/*
-				b.Log.Info("authupdate", e)
-				b.Log.Info("hash", e.Hash)
-				ioutil.WriteFile("sentry", e.Hash, 0666)
-			*/
-		case *steam.LogOnFailedEvent:
-			b.Log.Info("Logon failed", e)
-			switch e.Result {
-			case steamlang.EResult_AccountLogonDeniedNeedTwoFactorCode:
-				{
-					b.Log.Info("Steam guard isn't letting me in! Enter 2FA code:")
-					var code string
-					fmt.Scanf("%s", &code)
-					myLoginInfo.TwoFactorCode = code
-				}
-			case steamlang.EResult_AccountLogonDenied:
-				{
-					b.Log.Info("Steam guard isn't letting me in! Enter auth code:")
-					var code string
-					fmt.Scanf("%s", &code)
-					myLoginInfo.AuthCode = code
-				}
-			default:
-				b.Log.Errorf("LogOnFailedEvent: %#v ", e.Result)
-				// TODO: Handle EResult_InvalidLoginAuthCode
-				return
-			}
-		case *steam.LoggedOnEvent:
-			b.Log.Debugf("LoggedOnEvent: %#v", e)
-			b.connected <- struct{}{}
-			b.Log.Debugf("setting online")
-			b.c.Social.SetPersonaState(steamlang.EPersonaState_Online)
-		case *steam.DisconnectedEvent:
-			b.Log.Info("Disconnected")
-			b.Log.Info("Attempting to reconnect...")
-			b.c.Connect()
-		case steam.FatalErrorEvent:
-			b.Log.Error(e)
-		case error:
-			b.Log.Error(e)
-		default:
-			b.Log.Debugf("unknown event %#v", e)
-		}
-	}
-}

bridge/telegram/handlers.go

@@ -0,0 +1,447 @@
+package btelegram
+
+import (
+	"html"
+	"regexp"
+	"strconv"
+	"strings"
+	"unicode/utf16"
+
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/42wim/matterbridge/bridge/helper"
+	tgbotapi "github.com/go-telegram-bot-api/telegram-bot-api"
+)
+
+func (b *Btelegram) handleUpdate(rmsg *config.Message, message, posted, edited *tgbotapi.Message) *tgbotapi.Message {
+	// handle channels
+	if posted != nil {
+		message = posted
+		rmsg.Text = message.Text
+	}
+
+	// edited channel message
+	if edited != nil && !b.GetBool("EditDisable") {
+		message = edited
+		rmsg.Text = rmsg.Text + message.Text + b.GetString("EditSuffix")
+	}
+	return message
+}
+
+// handleChannels checks if it's a channel message and if the message is a new or edited messages
+func (b *Btelegram) handleChannels(rmsg *config.Message, message *tgbotapi.Message, update tgbotapi.Update) *tgbotapi.Message {
+	return b.handleUpdate(rmsg, message, update.ChannelPost, update.EditedChannelPost)
+}
+
+// handleGroups checks if it's a group message and if the message is a new or edited messages
+func (b *Btelegram) handleGroups(rmsg *config.Message, message *tgbotapi.Message, update tgbotapi.Update) *tgbotapi.Message {
+	return b.handleUpdate(rmsg, message, update.Message, update.EditedMessage)
+}
+
+// handleForwarded handles forwarded messages
+func (b *Btelegram) handleForwarded(rmsg *config.Message, message *tgbotapi.Message) {
+	if message.ForwardDate == 0 {
+		return
+	}
+
+	if message.ForwardFrom == nil {
+		rmsg.Text = "Forwarded from " + unknownUser + ": " + rmsg.Text
+		return
+	}
+
+	usernameForward := ""
+	if b.GetBool("UseFirstName") {
+		usernameForward = message.ForwardFrom.FirstName
+	}
+
+	if usernameForward == "" {
+		usernameForward = message.ForwardFrom.UserName
+		if usernameForward == "" {
+			usernameForward = message.ForwardFrom.FirstName
+		}
+	}
+
+	if usernameForward == "" {
+		usernameForward = unknownUser
+	}
+
+	rmsg.Text = "Forwarded from " + usernameForward + ": " + rmsg.Text
+}
+
+// handleQuoting handles quoting of previous messages
+func (b *Btelegram) handleQuoting(rmsg *config.Message, message *tgbotapi.Message) {
+	if message.ReplyToMessage != nil {
+		usernameReply := ""
+		if message.ReplyToMessage.From != nil {
+			if b.GetBool("UseFirstName") {
+				usernameReply = message.ReplyToMessage.From.FirstName
+			}
+			if usernameReply == "" {
+				usernameReply = message.ReplyToMessage.From.UserName
+				if usernameReply == "" {
+					usernameReply = message.ReplyToMessage.From.FirstName
+				}
+			}
+		}
+		if usernameReply == "" {
+			usernameReply = unknownUser
+		}
+		if !b.GetBool("QuoteDisable") {
+			rmsg.Text = b.handleQuote(rmsg.Text, usernameReply, message.ReplyToMessage.Text)
+		}
+	}
+}
+
+// handleUsername handles the correct setting of the username
+func (b *Btelegram) handleUsername(rmsg *config.Message, message *tgbotapi.Message) {
+	if message.From != nil {
+		rmsg.UserID = strconv.Itoa(message.From.ID)
+		if b.GetBool("UseFirstName") {
+			rmsg.Username = message.From.FirstName
+		}
+		if rmsg.Username == "" {
+			rmsg.Username = message.From.UserName
+			if rmsg.Username == "" {
+				rmsg.Username = message.From.FirstName
+			}
+		}
+		// only download avatars if we have a place to upload them (configured mediaserver)
+		if b.General.MediaServerUpload != "" || (b.General.MediaServerDownload != "" && b.General.MediaDownloadPath != "") {
+			b.handleDownloadAvatar(message.From.ID, rmsg.Channel)
+		}
+	}
+
+	// if we really didn't find a username, set it to unknown
+	if rmsg.Username == "" {
+		rmsg.Username = unknownUser
+	}
+}
+
+func (b *Btelegram) handleRecv(updates <-chan tgbotapi.Update) {
+	for update := range updates {
+		b.Log.Debugf("== Receiving event: %#v", update.Message)
+
+		if update.Message == nil && update.ChannelPost == nil &&
+			update.EditedMessage == nil && update.EditedChannelPost == nil {
+			b.Log.Error("Getting nil messages, this shouldn't happen.")
+			continue
+		}
+
+		var message *tgbotapi.Message
+
+		rmsg := config.Message{Account: b.Account, Extra: make(map[string][]interface{})}
+
+		// handle channels
+		message = b.handleChannels(&rmsg, message, update)
+
+		// handle groups
+		message = b.handleGroups(&rmsg, message, update)
+
+		if message == nil {
+			b.Log.Error("message is nil, this shouldn't happen.")
+			continue
+		}
+
+		// set the ID's from the channel or group message
+		rmsg.ID = strconv.Itoa(message.MessageID)
+		rmsg.Channel = strconv.FormatInt(message.Chat.ID, 10)
+
+		// handle username
+		b.handleUsername(&rmsg, message)
+
+		// handle any downloads
+		err := b.handleDownload(&rmsg, message)
+		if err != nil {
+			b.Log.Errorf("download failed: %s", err)
+		}
+
+		// handle forwarded messages
+		b.handleForwarded(&rmsg, message)
+
+		// quote the previous message
+		b.handleQuoting(&rmsg, message)
+
+		// handle entities (adding URLs)
+		b.handleEntities(&rmsg, message)
+
+		if rmsg.Text != "" || len(rmsg.Extra) > 0 {
+			rmsg.Text = helper.RemoveEmptyNewLines(rmsg.Text)
+			// channels don't have (always?) user information. see #410
+			if message.From != nil {
+				rmsg.Avatar = helper.GetAvatar(b.avatarMap, strconv.Itoa(message.From.ID), b.General)
+			}
+
+			b.Log.Debugf("<= Sending message from %s on %s to gateway", rmsg.Username, b.Account)
+			b.Log.Debugf("<= Message is %#v", rmsg)
+			b.Remote <- rmsg
+		}
+	}
+}
+
+// handleDownloadAvatar downloads the avatar of userid from channel
+// sends a EVENT_AVATAR_DOWNLOAD message to the gateway if successful.
+// logs an error message if it fails
+func (b *Btelegram) handleDownloadAvatar(userid int, channel string) {
+	rmsg := config.Message{Username: "system",
+		Text:    "avatar",
+		Channel: channel,
+		Account: b.Account,
+		UserID:  strconv.Itoa(userid),
+		Event:   config.EventAvatarDownload,
+		Extra:   make(map[string][]interface{})}
+
+	if _, ok := b.avatarMap[strconv.Itoa(userid)]; !ok {
+		photos, err := b.c.GetUserProfilePhotos(tgbotapi.UserProfilePhotosConfig{UserID: userid, Limit: 1})
+		if err != nil {
+			b.Log.Errorf("Userprofile download failed for %#v %s", userid, err)
+		}
+
+		if len(photos.Photos) > 0 {
+			photo := photos.Photos[0][0]
+			url := b.getFileDirectURL(photo.FileID)
+			name := strconv.Itoa(userid) + ".png"
+			b.Log.Debugf("trying to download %#v fileid %#v with size %#v", name, photo.FileID, photo.FileSize)
+
+			err := helper.HandleDownloadSize(b.Log, &rmsg, name, int64(photo.FileSize), b.General)
+			if err != nil {
+				b.Log.Error(err)
+				return
+			}
+			data, err := helper.DownloadFile(url)
+			if err != nil {
+				b.Log.Errorf("download %s failed %#v", url, err)
+				return
+			}
+			helper.HandleDownloadData(b.Log, &rmsg, name, rmsg.Text, "", data, b.General)
+			b.Remote <- rmsg
+		}
+	}
+}
+
+func (b *Btelegram) maybeConvertTgs(name *string, data *[]byte) {
+	var format string
+	switch b.GetString("MediaConvertTgs") {
+	case FormatWebp:
+		b.Log.Debugf("Tgs to WebP conversion enabled, converting %v", name)
+		format = FormatWebp
+	case FormatPng:
+		// The WebP to PNG converter can't handle animated webp files yet,
+		// and I'm not going to write a path for x/image/webp.
+		// The error message would be:
+		//     conversion failed: webp: non-Alpha VP8X is not implemented
+		// So instead, we tell lottie to directly go to PNG.
+		b.Log.Debugf("Tgs to PNG conversion enabled, converting %v", name)
+		format = FormatPng
+	default:
+		// Otherwise, no conversion was requested. Trying to run the usual webp
+		// converter would fail, because '.tgs.webp' is actually a gzipped JSON
+		// file, and has nothing to do with WebP.
+		return
+	}
+	err := helper.ConvertTgsToX(data, format, b.Log)
+	if err != nil {
+		b.Log.Errorf("conversion failed: %v", err)
+	} else {
+		*name = strings.Replace(*name, "tgs.webp", format, 1)
+	}
+}
+
+func (b *Btelegram) maybeConvertWebp(name *string, data *[]byte) {
+	if b.GetBool("MediaConvertWebPToPNG") {
+		b.Log.Debugf("WebP to PNG conversion enabled, converting %v", name)
+		err := helper.ConvertWebPToPNG(data)
+		if err != nil {
+			b.Log.Errorf("conversion failed: %v", err)
+		} else {
+			*name = strings.Replace(*name, ".webp", ".png", 1)
+		}
+	}
+}
+
+// handleDownloadFile handles file download
+func (b *Btelegram) handleDownload(rmsg *config.Message, message *tgbotapi.Message) error {
+	size := 0
+	var url, name, text string
+	switch {
+	case message.Sticker != nil:
+		text, name, url = b.getDownloadInfo(message.Sticker.FileID, ".webp", true)
+		size = message.Sticker.FileSize
+	case message.Voice != nil:
+		text, name, url = b.getDownloadInfo(message.Voice.FileID, ".ogg", true)
+		size = message.Voice.FileSize
+	case message.Video != nil:
+		text, name, url = b.getDownloadInfo(message.Video.FileID, "", true)
+		size = message.Video.FileSize
+	case message.Audio != nil:
+		text, name, url = b.getDownloadInfo(message.Audio.FileID, "", true)
+		size = message.Audio.FileSize
+	case message.Document != nil:
+		_, _, url = b.getDownloadInfo(message.Document.FileID, "", false)
+		size = message.Document.FileSize
+		name = message.Document.FileName
+		text = " " + message.Document.FileName + " : " + url
+	case message.Photo != nil:
+		photos := *message.Photo
+		size = photos[len(photos)-1].FileSize
+		text, name, url = b.getDownloadInfo(photos[len(photos)-1].FileID, "", true)
+	}
+
+	// if name is empty we didn't match a thing to download
+	if name == "" {
+		return nil
+	}
+	// use the URL instead of native upload
+	if b.GetBool("UseInsecureURL") {
+		b.Log.Debugf("Setting message text to :%s", text)
+		rmsg.Text += text
+		return nil
+	}
+	// if we have a file attached, download it (in memory) and put a pointer to it in msg.Extra
+	err := helper.HandleDownloadSize(b.Log, rmsg, name, int64(size), b.General)
+	if err != nil {
+		return err
+	}
+	data, err := helper.DownloadFile(url)
+	if err != nil {
+		return err
+	}
+
+	if strings.HasSuffix(name, ".tgs.webp") {
+		b.maybeConvertTgs(&name, data)
+	} else if strings.HasSuffix(name, ".webp") {
+		b.maybeConvertWebp(&name, data)
+	}
+
+	helper.HandleDownloadData(b.Log, rmsg, name, message.Caption, "", data, b.General)
+	return nil
+}
+
+func (b *Btelegram) getDownloadInfo(id string, suffix string, urlpart bool) (string, string, string) {
+	url := b.getFileDirectURL(id)
+	name := ""
+	if urlpart {
+		urlPart := strings.Split(url, "/")
+		name = urlPart[len(urlPart)-1]
+	}
+	if suffix != "" && !strings.HasSuffix(name, suffix) {
+		name += suffix
+	}
+	text := " " + url
+	return text, name, url
+}
+
+// handleDelete handles message deleting
+func (b *Btelegram) handleDelete(msg *config.Message, chatid int64) (string, error) {
+	if msg.ID == "" {
+		return "", nil
+	}
+	msgid, err := strconv.Atoi(msg.ID)
+	if err != nil {
+		return "", err
+	}
+	_, err = b.c.DeleteMessage(tgbotapi.DeleteMessageConfig{ChatID: chatid, MessageID: msgid})
+	return "", err
+}
+
+// handleEdit handles message editing.
+func (b *Btelegram) handleEdit(msg *config.Message, chatid int64) (string, error) {
+	msgid, err := strconv.Atoi(msg.ID)
+	if err != nil {
+		return "", err
+	}
+	if strings.ToLower(b.GetString("MessageFormat")) == HTMLNick {
+		b.Log.Debug("Using mode HTML - nick only")
+		msg.Text = html.EscapeString(msg.Text)
+	}
+	m := tgbotapi.NewEditMessageText(chatid, msgid, msg.Username+msg.Text)
+	switch b.GetString("MessageFormat") {
+	case HTMLFormat:
+		b.Log.Debug("Using mode HTML")
+		m.ParseMode = tgbotapi.ModeHTML
+	case "Markdown":
+		b.Log.Debug("Using mode markdown")
+		m.ParseMode = tgbotapi.ModeMarkdown
+	case MarkdownV2:
+		b.Log.Debug("Using mode MarkdownV2")
+		m.ParseMode = MarkdownV2
+	}
+	if strings.ToLower(b.GetString("MessageFormat")) == HTMLNick {
+		b.Log.Debug("Using mode HTML - nick only")
+		m.ParseMode = tgbotapi.ModeHTML
+	}
+	_, err = b.c.Send(m)
+	if err != nil {
+		return "", err
+	}
+	return "", nil
+}
+
+// handleUploadFile handles native upload of files
+func (b *Btelegram) handleUploadFile(msg *config.Message, chatid int64) string {
+	var c tgbotapi.Chattable
+	for _, f := range msg.Extra["file"] {
+		fi := f.(config.FileInfo)
+		file := tgbotapi.FileBytes{
+			Name:  fi.Name,
+			Bytes: *fi.Data,
+		}
+		re := regexp.MustCompile(".(jpg|png)$")
+		if re.MatchString(fi.Name) {
+			c = tgbotapi.NewPhotoUpload(chatid, file)
+		} else {
+			c = tgbotapi.NewDocumentUpload(chatid, file)
+		}
+		_, err := b.c.Send(c)
+		if err != nil {
+			b.Log.Errorf("file upload failed: %#v", err)
+		}
+		if fi.Comment != "" {
+			if _, err := b.sendMessage(chatid, msg.Username, fi.Comment); err != nil {
+				b.Log.Errorf("posting file comment %s failed: %s", fi.Comment, err)
+			}
+		}
+	}
+	return ""
+}
+
+func (b *Btelegram) handleQuote(message, quoteNick, quoteMessage string) string {
+	format := b.GetString("quoteformat")
+	if format == "" {
+		format = "{MESSAGE} (re @{QUOTENICK}: {QUOTEMESSAGE})"
+	}
+	quoteMessagelength := len(quoteMessage)
+	if b.GetInt("QuoteLengthLimit") != 0 && quoteMessagelength >= b.GetInt("QuoteLengthLimit") {
+		runes := []rune(quoteMessage)
+		quoteMessage = string(runes[0:b.GetInt("QuoteLengthLimit")])
+		if quoteMessagelength > b.GetInt("QuoteLengthLimit") {
+			quoteMessage += "..."
+		}
+	}
+	format = strings.Replace(format, "{MESSAGE}", message, -1)
+	format = strings.Replace(format, "{QUOTENICK}", quoteNick, -1)
+	format = strings.Replace(format, "{QUOTEMESSAGE}", quoteMessage, -1)
+	return format
+}
+
+// handleEntities handles messageEntities
+func (b *Btelegram) handleEntities(rmsg *config.Message, message *tgbotapi.Message) {
+	if message.Entities == nil {
+		return
+	}
+	// for now only do URL replacements
+	for _, e := range *message.Entities {
+		if e.Type == "text_link" {
+			url, err := e.ParseURL()
+			if err != nil {
+				b.Log.Errorf("entity text_link url parse failed: %s", err)
+				continue
+			}
+			utfEncodedString := utf16.Encode([]rune(rmsg.Text))
+			if e.Offset+e.Length > len(utfEncodedString) {
+				b.Log.Errorf("entity length is too long %d > %d", e.Offset+e.Length, len(utfEncodedString))
+				continue
+			}
+			link := utf16.Decode(utfEncodedString[e.Offset : e.Offset+e.Length])
+			rmsg.Text = strings.Replace(rmsg.Text, string(link), url.String(), 1)
+		}
+	}
+}

bridge/telegram/html.go

@@ -33,7 +33,7 @@ func (options *customHTML) Header(out *bytes.Buffer, text func() bool, level int
 }
 
 func (options *customHTML) HRule(out *bytes.Buffer) {
-	out.WriteByte('\n')
+	out.WriteByte('\n') //nolint:errcheck
 }
 
 func (options *customHTML) BlockQuote(out *bytes.Buffer, text []byte) {

bridge/telegram/telegram.go

@@ -2,14 +2,23 @@ package btelegram
 
 import (
 	"html"
-	"regexp"
+	"log"
 	"strconv"
 	"strings"
 
 	"github.com/42wim/matterbridge/bridge"
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/42wim/matterbridge/bridge/helper"
-	"github.com/go-telegram-bot-api/telegram-bot-api"
+	tgbotapi "github.com/go-telegram-bot-api/telegram-bot-api"
+)
+
+const (
+	unknownUser = "unknown"
+	HTMLFormat  = "HTML"
+	HTMLNick    = "htmlnick"
+	MarkdownV2  = "MarkdownV2"
+	FormatPng   = "png"
+	FormatWebp  = "webp"
 )
 
 type Btelegram struct {
@@ -19,6 +28,16 @@ type Btelegram struct {
 }
 
 func New(cfg *bridge.Config) bridge.Bridger {
+	tgsConvertFormat := cfg.GetString("MediaConvertTgs")
+	if tgsConvertFormat != "" {
+		err := helper.CanConvertTgsToX()
+		if err != nil {
+			log.Fatalf("Telegram bridge configured to convert .tgs files to '%s', but lottie does not appear to work:\n%#v", tgsConvertFormat, err)
+		}
+		if tgsConvertFormat != FormatPng && tgsConvertFormat != FormatWebp {
+			log.Fatalf("Telegram bridge configured to convert .tgs files to '%s', but only '%s' and '%s' are supported.", FormatPng, FormatWebp, tgsConvertFormat)
+		}
+	}
 	return &Btelegram{Config: cfg, avatarMap: make(map[string]string)}
 }
 
@@ -60,31 +79,25 @@ func (b *Btelegram) Send(msg config.Message) (string, error) {
 	}
 
 	// map the file SHA to our user (caches the avatar)
-	if msg.Event == config.EVENT_AVATAR_DOWNLOAD {
+	if msg.Event == config.EventAvatarDownload {
 		return b.cacheAvatar(&msg)
 	}
 
-	if b.GetString("MessageFormat") == "HTML" {
+	if b.GetString("MessageFormat") == HTMLFormat {
 		msg.Text = makeHTML(msg.Text)
 	}
 
 	// Delete message
-	if msg.Event == config.EVENT_MSG_DELETE {
+	if msg.Event == config.EventMsgDelete {
-		if msg.ID == "" {
+		return b.handleDelete(&msg, chatid)
-			return "", nil
-		}
-		msgid, err := strconv.Atoi(msg.ID)
-		if err != nil {
-			return "", err
-		}
-		_, err = b.c.DeleteMessage(tgbotapi.DeleteMessageConfig{ChatID: chatid, MessageID: msgid})
-		return "", err
 	}
 
 	// Upload a file if it exists
 	if msg.Extra != nil {
 		for _, rmsg := range helper.HandleExtra(&msg, b.General) {
-			b.sendMessage(chatid, rmsg.Username, rmsg.Text)
+			if _, msgErr := b.sendMessage(chatid, rmsg.Username, rmsg.Text); msgErr != nil {
+				b.Log.Errorf("sendMessage failed: %s", msgErr)
+			}
 		}
 		// check if we have files to upload (from slack, telegram or mattermost)
 		if len(msg.Extra["file"]) > 0 {
@@ -94,160 +107,18 @@ func (b *Btelegram) Send(msg config.Message) (string, error) {
 
 	// edit the message if we have a msg ID
 	if msg.ID != "" {
-		msgid, err := strconv.Atoi(msg.ID)
+		return b.handleEdit(&msg, chatid)
-		if err != nil {
-			return "", err
-		}
-		if strings.ToLower(b.GetString("MessageFormat")) == "htmlnick" {
-			b.Log.Debug("Using mode HTML - nick only")
-			msg.Text = html.EscapeString(msg.Text)
-		}
-		m := tgbotapi.NewEditMessageText(chatid, msgid, msg.Username+msg.Text)
-		if b.GetString("MessageFormat") == "HTML" {
-			b.Log.Debug("Using mode HTML")
-			m.ParseMode = tgbotapi.ModeHTML
-		}
-		if b.GetString("MessageFormat") == "Markdown" {
-			b.Log.Debug("Using mode markdown")
-			m.ParseMode = tgbotapi.ModeMarkdown
-		}
-		if strings.ToLower(b.GetString("MessageFormat")) == "htmlnick" {
-			b.Log.Debug("Using mode HTML - nick only")
-			m.ParseMode = tgbotapi.ModeHTML
-		}
-		_, err = b.c.Send(m)
-		if err != nil {
-			return "", err
-		}
-		return "", nil
 	}
 
 	// Post normal message
-	return b.sendMessage(chatid, msg.Username, msg.Text)
+	// TODO: recheck it.
-}
+	// Ignore empty text field needs for prevent double messages from whatsapp to telegram
-
+	// when sending media with text caption
-func (b *Btelegram) handleRecv(updates <-chan tgbotapi.Update) {
+	if msg.Text != "" {
-	for update := range updates {
+		return b.sendMessage(chatid, msg.Username, msg.Text)
-		b.Log.Debugf("== Receiving event: %#v", update.Message)
-
-		if update.Message == nil && update.ChannelPost == nil && update.EditedMessage == nil && update.EditedChannelPost == nil {
-			b.Log.Error("Getting nil messages, this shouldn't happen.")
-			continue
-		}
-
-		var message *tgbotapi.Message
-
-		rmsg := config.Message{Account: b.Account, Extra: make(map[string][]interface{})}
-
-		// handle channels
-		if update.ChannelPost != nil {
-			message = update.ChannelPost
-			rmsg.Text = message.Text
-		}
-
-		// edited channel message
-		if update.EditedChannelPost != nil && !b.GetBool("EditDisable") {
-			message = update.EditedChannelPost
-			rmsg.Text = rmsg.Text + message.Text + b.GetString("EditSuffix")
-		}
-
-		// handle groups
-		if update.Message != nil {
-			message = update.Message
-			rmsg.Text = message.Text
-		}
-
-		// edited group message
-		if update.EditedMessage != nil && !b.GetBool("EditDisable") {
-			message = update.EditedMessage
-			rmsg.Text = rmsg.Text + message.Text + b.GetString("EditSuffix")
-		}
-
-		// set the ID's from the channel or group message
-		rmsg.ID = strconv.Itoa(message.MessageID)
-		rmsg.Channel = strconv.FormatInt(message.Chat.ID, 10)
-
-		// handle username
-		if message.From != nil {
-			rmsg.UserID = strconv.Itoa(message.From.ID)
-			if b.GetBool("UseFirstName") {
-				rmsg.Username = message.From.FirstName
-			}
-			if rmsg.Username == "" {
-				rmsg.Username = message.From.UserName
-				if rmsg.Username == "" {
-					rmsg.Username = message.From.FirstName
-				}
-			}
-			// only download avatars if we have a place to upload them (configured mediaserver)
-			if b.General.MediaServerUpload != "" {
-				b.handleDownloadAvatar(message.From.ID, rmsg.Channel)
-			}
-		}
-
-		// if we really didn't find a username, set it to unknown
-		if rmsg.Username == "" {
-			rmsg.Username = "unknown"
-		}
-
-		// handle any downloads
-		err := b.handleDownload(message, &rmsg)
-		if err != nil {
-			b.Log.Errorf("download failed: %s", err)
-		}
-
-		// handle forwarded messages
-		if message.ForwardFrom != nil {
-			usernameForward := ""
-			if b.GetBool("UseFirstName") {
-				usernameForward = message.ForwardFrom.FirstName
-			}
-			if usernameForward == "" {
-				usernameForward = message.ForwardFrom.UserName
-				if usernameForward == "" {
-					usernameForward = message.ForwardFrom.FirstName
-				}
-			}
-			if usernameForward == "" {
-				usernameForward = "unknown"
-			}
-			rmsg.Text = "Forwarded from " + usernameForward + ": " + rmsg.Text
-		}
-
-		// quote the previous message
-		if message.ReplyToMessage != nil {
-			usernameReply := ""
-			if message.ReplyToMessage.From != nil {
-				if b.GetBool("UseFirstName") {
-					usernameReply = message.ReplyToMessage.From.FirstName
-				}
-				if usernameReply == "" {
-					usernameReply = message.ReplyToMessage.From.UserName
-					if usernameReply == "" {
-						usernameReply = message.ReplyToMessage.From.FirstName
-					}
-				}
-			}
-			if usernameReply == "" {
-				usernameReply = "unknown"
-			}
-			if !b.GetBool("QuoteDisable") {
-				rmsg.Text = b.handleQuote(rmsg.Text, usernameReply, message.ReplyToMessage.Text)
-			}
-		}
-
-		if rmsg.Text != "" || len(rmsg.Extra) > 0 {
-			rmsg.Text = helper.RemoveEmptyNewLines(rmsg.Text)
-			// channels don't have (always?) user information. see #410
-			if message.From != nil {
-				rmsg.Avatar = helper.GetAvatar(b.avatarMap, strconv.Itoa(message.From.ID), b.General)
-			}
-
-			b.Log.Debugf("<= Sending message from %s on %s to gateway", rmsg.Username, b.Account)
-			b.Log.Debugf("<= Message is %#v", rmsg)
-			b.Remote <- rmsg
-		}
 	}
+
+	return "", nil
 }
 
 func (b *Btelegram) getFileDirectURL(id string) string {
@@ -258,147 +129,10 @@ func (b *Btelegram) getFileDirectURL(id string) string {
 	return res
 }
 
-// handleDownloadAvatar downloads the avatar of userid from channel
-// sends a EVENT_AVATAR_DOWNLOAD message to the gateway if successful.
-// logs an error message if it fails
-func (b *Btelegram) handleDownloadAvatar(userid int, channel string) {
-	rmsg := config.Message{Username: "system", Text: "avatar", Channel: channel, Account: b.Account, UserID: strconv.Itoa(userid), Event: config.EVENT_AVATAR_DOWNLOAD, Extra: make(map[string][]interface{})}
-	if _, ok := b.avatarMap[strconv.Itoa(userid)]; !ok {
-		photos, err := b.c.GetUserProfilePhotos(tgbotapi.UserProfilePhotosConfig{UserID: userid, Limit: 1})
-		if err != nil {
-			b.Log.Errorf("Userprofile download failed for %#v %s", userid, err)
-		}
-
-		if len(photos.Photos) > 0 {
-			photo := photos.Photos[0][0]
-			url := b.getFileDirectURL(photo.FileID)
-			name := strconv.Itoa(userid) + ".png"
-			b.Log.Debugf("trying to download %#v fileid %#v with size %#v", name, photo.FileID, photo.FileSize)
-
-			err := helper.HandleDownloadSize(b.Log, &rmsg, name, int64(photo.FileSize), b.General)
-			if err != nil {
-				b.Log.Error(err)
-				return
-			}
-			data, err := helper.DownloadFile(url)
-			if err != nil {
-				b.Log.Errorf("download %s failed %#v", url, err)
-				return
-			}
-			helper.HandleDownloadData(b.Log, &rmsg, name, rmsg.Text, "", data, b.General)
-			b.Remote <- rmsg
-		}
-	}
-}
-
-// handleDownloadFile handles file download
-func (b *Btelegram) handleDownload(message *tgbotapi.Message, rmsg *config.Message) error {
-	size := 0
-	var url, name, text string
-
-	if message.Sticker != nil {
-		v := message.Sticker
-		size = v.FileSize
-		url = b.getFileDirectURL(v.FileID)
-		urlPart := strings.Split(url, "/")
-		name = urlPart[len(urlPart)-1]
-		if !strings.HasSuffix(name, ".webp") {
-			name = name + ".webp"
-		}
-		text = " " + url
-	}
-	if message.Video != nil {
-		v := message.Video
-		size = v.FileSize
-		url = b.getFileDirectURL(v.FileID)
-		urlPart := strings.Split(url, "/")
-		name = urlPart[len(urlPart)-1]
-		text = " " + url
-	}
-	if message.Photo != nil {
-		photos := *message.Photo
-		size = photos[len(photos)-1].FileSize
-		url = b.getFileDirectURL(photos[len(photos)-1].FileID)
-		urlPart := strings.Split(url, "/")
-		name = urlPart[len(urlPart)-1]
-		text = " " + url
-	}
-	if message.Document != nil {
-		v := message.Document
-		size = v.FileSize
-		url = b.getFileDirectURL(v.FileID)
-		name = v.FileName
-		text = " " + v.FileName + " : " + url
-	}
-	if message.Voice != nil {
-		v := message.Voice
-		size = v.FileSize
-		url = b.getFileDirectURL(v.FileID)
-		urlPart := strings.Split(url, "/")
-		name = urlPart[len(urlPart)-1]
-		text = " " + url
-		if !strings.HasSuffix(name, ".ogg") {
-			name = name + ".ogg"
-		}
-	}
-	if message.Audio != nil {
-		v := message.Audio
-		size = v.FileSize
-		url = b.getFileDirectURL(v.FileID)
-		urlPart := strings.Split(url, "/")
-		name = urlPart[len(urlPart)-1]
-		text = " " + url
-	}
-	// if name is empty we didn't match a thing to download
-	if name == "" {
-		return nil
-	}
-	// use the URL instead of native upload
-	if b.GetBool("UseInsecureURL") {
-		b.Log.Debugf("Setting message text to :%s", text)
-		rmsg.Text = rmsg.Text + text
-		return nil
-	}
-	// if we have a file attached, download it (in memory) and put a pointer to it in msg.Extra
-	err := helper.HandleDownloadSize(b.Log, rmsg, name, int64(size), b.General)
-	if err != nil {
-		return err
-	}
-	data, err := helper.DownloadFile(url)
-	if err != nil {
-		return err
-	}
-	helper.HandleDownloadData(b.Log, rmsg, name, message.Caption, "", data, b.General)
-	return nil
-}
-
-// handleUploadFile handles native upload of files
-func (b *Btelegram) handleUploadFile(msg *config.Message, chatid int64) (string, error) {
-	var c tgbotapi.Chattable
-	for _, f := range msg.Extra["file"] {
-		fi := f.(config.FileInfo)
-		file := tgbotapi.FileBytes{fi.Name, *fi.Data}
-		re := regexp.MustCompile(".(jpg|png)$")
-		if re.MatchString(fi.Name) {
-			c = tgbotapi.NewPhotoUpload(chatid, file)
-		} else {
-			c = tgbotapi.NewDocumentUpload(chatid, file)
-		}
-		_, err := b.c.Send(c)
-		if err != nil {
-			b.Log.Errorf("file upload failed: %#v", err)
-		}
-		if fi.Comment != "" {
-			b.sendMessage(chatid, msg.Username, fi.Comment)
-		}
-	}
-	return "", nil
-}
-
 func (b *Btelegram) sendMessage(chatid int64, username, text string) (string, error) {
 	m := tgbotapi.NewMessage(chatid, "")
 	m.Text = username + text
-	if b.GetString("MessageFormat") == "HTML" {
+	if b.GetString("MessageFormat") == HTMLFormat {
 		b.Log.Debug("Using mode HTML")
 		m.ParseMode = tgbotapi.ModeHTML
 	}
@@ -406,11 +140,18 @@ func (b *Btelegram) sendMessage(chatid int64, username, text string) (string, er
 		b.Log.Debug("Using mode markdown")
 		m.ParseMode = tgbotapi.ModeMarkdown
 	}
-	if strings.ToLower(b.GetString("MessageFormat")) == "htmlnick" {
+	if b.GetString("MessageFormat") == MarkdownV2 {
+		b.Log.Debug("Using mode MarkdownV2")
+		m.ParseMode = MarkdownV2
+	}
+	if strings.ToLower(b.GetString("MessageFormat")) == HTMLNick {
 		b.Log.Debug("Using mode HTML - nick only")
 		m.Text = username + html.EscapeString(text)
 		m.ParseMode = tgbotapi.ModeHTML
 	}
+
+	m.DisableWebPagePreview = b.GetBool("DisableWebPagePreview")
+
 	res, err := b.c.Send(m)
 	if err != nil {
 		return "", err
@@ -428,14 +169,3 @@ func (b *Btelegram) cacheAvatar(msg *config.Message) (string, error) {
 	}
 	return "", nil
 }
-
-func (b *Btelegram) handleQuote(message, quoteNick, quoteMessage string) string {
-	format := b.GetString("quoteformat")
-	if format == "" {
-		format = "{MESSAGE} (re @{QUOTENICK}: {QUOTEMESSAGE})"
-	}
-	format = strings.Replace(format, "{MESSAGE}", message, -1)
-	format = strings.Replace(format, "{QUOTENICK}", quoteNick, -1)
-	format = strings.Replace(format, "{QUOTEMESSAGE}", quoteMessage, -1)
-	return format
-}

bridge/whatsapp/handlers.go

@@ -0,0 +1,208 @@
+package bwhatsapp
+
+import (
+	"fmt"
+	"mime"
+	"strings"
+	"time"
+
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/42wim/matterbridge/bridge/helper"
+	"github.com/Rhymen/go-whatsapp"
+	"github.com/jpillora/backoff"
+)
+
+/*
+Implement handling messages coming from WhatsApp
+Check:
+- https://github.com/Rhymen/go-whatsapp#add-message-handlers
+- https://github.com/Rhymen/go-whatsapp/blob/master/handler.go
+- https://github.com/tulir/mautrix-whatsapp/tree/master/whatsapp-ext for more advanced command handling
+*/
+
+// HandleError received from WhatsApp
+func (b *Bwhatsapp) HandleError(err error) {
+	// ignore received invalid data errors. https://github.com/42wim/matterbridge/issues/843
+	// ignore tag 174 errors. https://github.com/42wim/matterbridge/issues/1094
+	if strings.Contains(err.Error(), "error processing data: received invalid data") || strings.Contains(err.Error(), "invalid string with tag 174") {
+		return
+	}
+
+	switch err.(type) {
+	case *whatsapp.ErrConnectionClosed, *whatsapp.ErrConnectionFailed:
+		b.reconnect(err)
+	default:
+		switch err {
+		case whatsapp.ErrConnectionTimeout:
+			b.reconnect(err)
+		default:
+			b.Log.Errorf("%v", err)
+		}
+	}
+}
+
+func (b *Bwhatsapp) reconnect(err error) {
+	bf := &backoff.Backoff{
+		Min:    time.Second,
+		Max:    5 * time.Minute,
+		Jitter: true,
+	}
+	for {
+		d := bf.Duration()
+		b.Log.Errorf("Connection failed, underlying error: %v", err)
+		b.Log.Infof("Waiting %s...", d)
+		time.Sleep(d)
+		b.Log.Info("Reconnecting...")
+		err := b.conn.Restore()
+		if err == nil {
+			bf.Reset()
+			b.startedAt = uint64(time.Now().Unix())
+			return
+		}
+	}
+}
+
+// HandleTextMessage sent from WhatsApp, relay it to the brige
+func (b *Bwhatsapp) HandleTextMessage(message whatsapp.TextMessage) {
+	if message.Info.FromMe { // || !strings.Contains(strings.ToLower(message.Text), "@echo") {
+		return
+	}
+	// whatsapp sends last messages to show context , cut them
+	if message.Info.Timestamp < b.startedAt {
+		return
+	}
+
+	messageTime := time.Unix(int64(message.Info.Timestamp), 0) // TODO check how behaves between timezones
+	groupJID := message.Info.RemoteJid
+
+	senderJID := message.Info.SenderJid
+	if len(senderJID) == 0 {
+		// TODO workaround till https://github.com/Rhymen/go-whatsapp/issues/86 resolved
+		if message.Info.Source != nil && message.Info.Source.Participant != nil {
+			senderJID = *message.Info.Source.Participant
+		}
+	}
+
+	// translate sender's JID to the nicest username we can get
+	senderName := b.getSenderName(senderJID)
+	if senderName == "" {
+		senderName = "Someone" // don't expose telephone number
+	}
+
+	extText := message.Info.Source.Message.ExtendedTextMessage
+	if extText != nil && extText.ContextInfo != nil && extText.ContextInfo.MentionedJid != nil {
+		// handle user mentions
+		for _, mentionedJID := range extText.ContextInfo.MentionedJid {
+			numberAndSuffix := strings.SplitN(mentionedJID, "@", 2)
+
+			// mentions comes as telephone numbers and we don't want to expose it to other bridges
+			// replace it with something more meaninful to others
+			mention := b.getSenderNotify(numberAndSuffix[0] + "@s.whatsapp.net")
+			if mention == "" {
+				mention = "someone"
+			}
+			message.Text = strings.Replace(message.Text, "@"+numberAndSuffix[0], "@"+mention, 1)
+		}
+	}
+
+	b.Log.Debugf("<= Sending message from %s on %s to gateway", senderJID, b.Account)
+	rmsg := config.Message{
+		UserID:    senderJID,
+		Username:  senderName,
+		Text:      message.Text,
+		Timestamp: messageTime,
+		Channel:   groupJID,
+		Account:   b.Account,
+		Protocol:  b.Protocol,
+		Extra:     make(map[string][]interface{}),
+		//	ParentID: TODO, // TODO handle thread replies  // map from Info.QuotedMessageID string
+		//	Event     string    `json:"event"`
+		//	Gateway   string  // will be added during message processing
+		ID: message.Info.Id}
+
+	if avatarURL, exists := b.userAvatars[senderJID]; exists {
+		rmsg.Avatar = avatarURL
+	}
+
+	b.Log.Debugf("<= Message is %#v", rmsg)
+	b.Remote <- rmsg
+}
+
+// HandleImageMessage sent from WhatsApp, relay it to the brige
+func (b *Bwhatsapp) HandleImageMessage(message whatsapp.ImageMessage) {
+	if message.Info.FromMe { // || !strings.Contains(strings.ToLower(message.Text), "@echo") {
+		return
+	}
+
+	// whatsapp sends last messages to show context , cut them
+	if message.Info.Timestamp < b.startedAt {
+		return
+	}
+
+	messageTime := time.Unix(int64(message.Info.Timestamp), 0) // TODO check how behaves between timezones
+	groupJID := message.Info.RemoteJid
+
+	senderJID := message.Info.SenderJid
+	// if len(senderJid) == 0 {
+	//   // TODO workaround till https://github.com/Rhymen/go-whatsapp/issues/86 resolved
+	//   senderJid = *message.Info.Source.Participant
+	// }
+
+	// translate sender's Jid to the nicest username we can get
+	senderName := b.getSenderName(senderJID)
+	if senderName == "" {
+		senderName = "Someone" // don't expose telephone number
+	}
+
+	b.Log.Debugf("<= Sending message from %s on %s to gateway", senderJID, b.Account)
+	rmsg := config.Message{
+		UserID:    senderJID,
+		Username:  senderName,
+		Timestamp: messageTime,
+		Channel:   groupJID,
+		Account:   b.Account,
+		Protocol:  b.Protocol,
+		Extra:     make(map[string][]interface{}),
+		//  ParentID: TODO,      // TODO handle thread replies  // map from Info.QuotedMessageID string
+		//  Event     string    `json:"event"`
+		//  Gateway   string     // will be added during message processing
+		ID: message.Info.Id}
+
+	if avatarURL, exists := b.userAvatars[senderJID]; exists {
+		rmsg.Avatar = avatarURL
+	}
+
+	// Download and unencrypt content
+	data, err := message.Download()
+	if err != nil {
+		b.Log.Errorf("%v", err)
+		return
+	}
+
+	// Get file extension by mimetype
+	fileExt, err := mime.ExtensionsByType(message.Type)
+	if err != nil {
+		b.Log.Errorf("%v", err)
+		return
+	}
+
+	filename := fmt.Sprintf("%v%v", message.Info.Id, fileExt[0])
+
+	b.Log.Debugf("<= Image downloaded and unencrypted")
+
+	// Move file to bridge storage
+	helper.HandleDownloadData(b.Log, &rmsg, filename, message.Caption, "", &data, b.General)
+
+	b.Log.Debugf("<= Image Message is %#v", rmsg)
+	b.Remote <- rmsg
+}
+
+//func (b *Bwhatsapp) HandleVideoMessage(message whatsapp.VideoMessage) {
+//	fmt.Println(message) // TODO implement
+//}
+//
+//func (b *Bwhatsapp) HandleJsonMessage(message string) {
+//	fmt.Println(message) // TODO implement
+//}
+// TODO HandleRawMessage
+// TODO HandleAudioMessage

bridge/whatsapp/helpers.go

@@ -0,0 +1,132 @@
+package bwhatsapp
+
+import (
+	"encoding/gob"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+
+	qrcodeTerminal "github.com/Baozisoftware/qrcode-terminal-go"
+	"github.com/Rhymen/go-whatsapp"
+)
+
+type ProfilePicInfo struct {
+	URL string `json:"eurl"`
+	Tag string `json:"tag"`
+
+	Status int16 `json:"status"`
+}
+
+func qrFromTerminal(invert bool) chan string {
+	qr := make(chan string)
+	go func() {
+		terminal := qrcodeTerminal.New()
+		if invert {
+			terminal = qrcodeTerminal.New2(qrcodeTerminal.ConsoleColors.BrightWhite, qrcodeTerminal.ConsoleColors.BrightBlack, qrcodeTerminal.QRCodeRecoveryLevels.Medium)
+		}
+
+		terminal.Get(<-qr).Print()
+	}()
+
+	return qr
+}
+
+func (b *Bwhatsapp) readSession() (whatsapp.Session, error) {
+	session := whatsapp.Session{}
+	sessionFile := b.Config.GetString(sessionFile)
+
+	if sessionFile == "" {
+		return session, errors.New("if you won't set SessionFile then you will need to scan QR code on every restart")
+	}
+
+	file, err := os.Open(sessionFile)
+	if err != nil {
+		return session, err
+	}
+	defer file.Close()
+	decoder := gob.NewDecoder(file)
+	err = decoder.Decode(&session)
+	if err != nil {
+		return session, err
+	}
+	return session, nil
+}
+
+func (b *Bwhatsapp) writeSession(session whatsapp.Session) error {
+	sessionFile := b.Config.GetString(sessionFile)
+
+	if sessionFile == "" {
+		// we already sent a warning while starting the bridge, so let's be quiet here
+		return nil
+	}
+
+	file, err := os.Create(sessionFile)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+	encoder := gob.NewEncoder(file)
+	err = encoder.Encode(session)
+
+	return err
+}
+
+func (b *Bwhatsapp) getSenderName(senderJid string) string {
+	if sender, exists := b.users[senderJid]; exists {
+		if sender.Name != "" {
+			return sender.Name
+		}
+		// if user is not in phone contacts
+		// it is the most obvious scenario unless you sync your phone contacts with some remote updated source
+		// users can change it in their WhatsApp settings -> profile -> click on Avatar
+		if sender.Notify != "" {
+			return sender.Notify
+		}
+
+		if sender.Short != "" {
+			return sender.Short
+		}
+	}
+
+	// try to reload this contact
+	_, err := b.conn.Contacts()
+	if err != nil {
+		b.Log.Errorf("error on update of contacts: %v", err)
+	}
+
+	if contact, exists := b.conn.Store.Contacts[senderJid]; exists {
+		// Add it to the user map
+		b.users[senderJid] = contact
+
+		if contact.Name != "" {
+			return contact.Name
+		}
+		// if user is not in phone contacts
+		// same as above
+		return contact.Notify
+	}
+
+	return ""
+}
+
+func (b *Bwhatsapp) getSenderNotify(senderJid string) string {
+	if sender, exists := b.users[senderJid]; exists {
+		return sender.Notify
+	}
+	return ""
+}
+
+func (b *Bwhatsapp) GetProfilePicThumb(jid string) (*ProfilePicInfo, error) {
+	data, err := b.conn.GetProfilePicThumb(jid)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get avatar: %v", err)
+	}
+	content := <-data
+	info := &ProfilePicInfo{}
+	err = json.Unmarshal([]byte(content), info)
+	if err != nil {
+		return info, fmt.Errorf("failed to unmarshal avatar info: %v", err)
+	}
+	return info, nil
+}

bridge/whatsapp/whatsapp.go

@@ -0,0 +1,368 @@
+package bwhatsapp
+
+import (
+	"bytes"
+	"crypto/rand"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"mime"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/42wim/matterbridge/bridge"
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/Rhymen/go-whatsapp"
+)
+
+const (
+	// Account config parameters
+	cfgNumber         = "Number"
+	qrOnWhiteTerminal = "QrOnWhiteTerminal"
+	sessionFile       = "SessionFile"
+)
+
+// Bwhatsapp Bridge structure keeping all the information needed for relying
+type Bwhatsapp struct {
+	*bridge.Config
+
+	// https://github.com/Rhymen/go-whatsapp/blob/c31092027237441cffba1b9cb148eadf7c83c3d2/session.go#L18-L21
+	session   *whatsapp.Session
+	conn      *whatsapp.Conn
+	startedAt uint64
+
+	users       map[string]whatsapp.Contact
+	userAvatars map[string]string
+}
+
+// New Create a new WhatsApp bridge. This will be called for each [whatsapp.<server>] entry you have in the config file
+func New(cfg *bridge.Config) bridge.Bridger {
+	number := cfg.GetString(cfgNumber)
+	if number == "" {
+		cfg.Log.Fatalf("Missing configuration for WhatsApp bridge: Number")
+	}
+
+	b := &Bwhatsapp{
+		Config: cfg,
+
+		users:       make(map[string]whatsapp.Contact),
+		userAvatars: make(map[string]string),
+	}
+	return b
+}
+
+// Connect to WhatsApp. Required implementation of the Bridger interface
+// https://github.com/42wim/matterbridge/blob/2cfd880cdb0df29771bf8f31df8d990ab897889d/bridge/bridge.go#L11-L16
+func (b *Bwhatsapp) Connect() error {
+	b.RLock() // TODO do we need locking for Whatsapp?
+	defer b.RUnlock()
+
+	number := b.GetString(cfgNumber)
+	if number == "" {
+		return errors.New("WhatsApp's telephone Number need to be configured")
+	}
+
+	// https://github.com/Rhymen/go-whatsapp#creating-a-connection
+	b.Log.Debugln("Connecting to WhatsApp..")
+	conn, err := whatsapp.NewConn(20 * time.Second)
+	conn.SetClientVersion(0, 4, 2080)
+	if err != nil {
+		return errors.New("failed to connect to WhatsApp: " + err.Error())
+	}
+
+	b.conn = conn
+
+	b.conn.AddHandler(b)
+	b.Log.Debugln("WhatsApp connection successful")
+
+	// load existing session in order to keep it between restarts
+	if b.session == nil {
+		var session whatsapp.Session
+		session, err = b.readSession()
+
+		if err == nil {
+			b.Log.Debugln("Restoring WhatsApp session..")
+
+			// https://github.com/Rhymen/go-whatsapp#restore
+			session, err = b.conn.RestoreWithSession(session)
+			if err != nil {
+				// TODO return or continue to normal login?
+				// restore session connection timed out (I couldn't get over it without logging in again)
+				return errors.New("failed to restore session: " + err.Error())
+			}
+
+			b.session = &session
+			b.Log.Debugln("Session restored successfully!")
+		} else {
+			b.Log.Warn(err.Error())
+		}
+	}
+
+	// login to a new session
+	if b.session == nil {
+		err = b.Login()
+		if err != nil {
+			return err
+		}
+	}
+	b.startedAt = uint64(time.Now().Unix())
+
+	_, err = b.conn.Contacts()
+	if err != nil {
+		return fmt.Errorf("error on update of contacts: %v", err)
+	}
+
+	// map all the users
+	for id, contact := range b.conn.Store.Contacts {
+		if !isGroupJid(id) && id != "status@broadcast" {
+			// it is user
+			b.users[id] = contact
+		}
+	}
+
+	// get user avatar asynchronously
+	go func() {
+		b.Log.Debug("Getting user avatars..")
+
+		for jid := range b.users {
+			info, err := b.GetProfilePicThumb(jid)
+			if err != nil {
+				b.Log.Warnf("Could not get profile photo of %s: %v", jid, err)
+
+			} else {
+				// TODO any race conditions here?
+				b.userAvatars[jid] = info.URL
+			}
+		}
+		b.Log.Debug("Finished getting avatars..")
+	}()
+
+	return nil
+}
+
+// Login to WhatsApp creating a new session. This will require to scan a QR code on your mobile device
+func (b *Bwhatsapp) Login() error {
+	b.Log.Debugln("Logging in..")
+
+	invert := b.GetBool(qrOnWhiteTerminal) // false is the default
+	qrChan := qrFromTerminal(invert)
+
+	session, err := b.conn.Login(qrChan)
+	if err != nil {
+		b.Log.Warnln("Failed to log in:", err)
+		return err
+	}
+	b.session = &session
+
+	b.Log.Infof("Logged into session: %#v", session)
+	b.Log.Infof("Connection: %#v", b.conn)
+
+	err = b.writeSession(session)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "error saving session: %v\n", err)
+	}
+
+	// TODO change connection strings to configured ones longClientName:"github.com/rhymen/go-whatsapp", shortClientName:"go-whatsapp"}" prefix=whatsapp
+	// TODO get also a nice logo
+
+	// TODO notification about unplugged and dead battery
+	// conn.Info: Wid, Pushname, Connected, Battery, Plugged
+
+	return nil
+}
+
+// Disconnect is called while reconnecting to the bridge
+// TODO 42wim Documentation would be helpful on when reconnects happen and what should be done in this function
+// Required implementation of the Bridger interface
+// https://github.com/42wim/matterbridge/blob/2cfd880cdb0df29771bf8f31df8d990ab897889d/bridge/bridge.go#L11-L16
+func (b *Bwhatsapp) Disconnect() error {
+	// We could Logout, but that would close the session completely and would require a new QR code scan
+	// https://github.com/Rhymen/go-whatsapp/blob/c31092027237441cffba1b9cb148eadf7c83c3d2/session.go#L377-L381
+	return nil
+}
+
+func isGroupJid(identifier string) bool {
+	return strings.HasSuffix(identifier, "@g.us") || strings.HasSuffix(identifier, "@temp") || strings.HasSuffix(identifier, "@broadcast")
+}
+
+// JoinChannel Join a WhatsApp group specified in gateway config as channel='number-id@g.us' or channel='Channel name'
+// Required implementation of the Bridger interface
+// https://github.com/42wim/matterbridge/blob/2cfd880cdb0df29771bf8f31df8d990ab897889d/bridge/bridge.go#L11-L16
+func (b *Bwhatsapp) JoinChannel(channel config.ChannelInfo) error {
+	byJid := isGroupJid(channel.Name)
+
+	// verify if we are member of the given group
+	if byJid {
+		// channel.Name specifies static group jID, not the name
+		if _, exists := b.conn.Store.Contacts[channel.Name]; !exists {
+			return fmt.Errorf("account doesn't belong to group with jid %s", channel.Name)
+		}
+	} else {
+		// channel.Name specifies group name that might change, warn about it
+		var jids []string
+		for id, contact := range b.conn.Store.Contacts {
+			if isGroupJid(id) && contact.Name == channel.Name {
+				jids = append(jids, id)
+			}
+		}
+
+		switch len(jids) {
+		case 0:
+			// didn't match any group - print out possibilites
+			// TODO sort
+			// copy b;
+			//sort.Slice(people, func(i, j int) bool {
+			//	return people[i].Age > people[j].Age
+			//})
+			for id, contact := range b.conn.Store.Contacts {
+				if isGroupJid(id) {
+					b.Log.Infof("%s %s", contact.Jid, contact.Name)
+				}
+			}
+			return fmt.Errorf("please specify group's JID from the list above instead of the name '%s'", channel.Name)
+
+		case 1:
+			return fmt.Errorf("group name might change. Please configure gateway with channel=\"%v\" instead of channel=\"%v\"", jids[0], channel.Name)
+
+		default:
+			return fmt.Errorf("there is more than one group with name '%s'. Please specify one of JIDs as channel name: %v", channel.Name, jids)
+		}
+	}
+
+	return nil
+}
+
+// Post a document message from the bridge to WhatsApp
+func (b *Bwhatsapp) PostDocumentMessage(msg config.Message, filetype string) (string, error) {
+	fi := msg.Extra["file"][0].(config.FileInfo)
+
+	// Post document message
+	message := whatsapp.DocumentMessage{
+		Info: whatsapp.MessageInfo{
+			RemoteJid: msg.Channel,
+		},
+		Title:    fi.Name,
+		FileName: fi.Name,
+		Type:     filetype,
+		Content:  bytes.NewReader(*fi.Data),
+	}
+
+	b.Log.Debugf("=> Sending %#v", msg)
+
+	// create message ID
+	// TODO follow and act if https://github.com/Rhymen/go-whatsapp/issues/101 implemented
+	idBytes := make([]byte, 10)
+	if _, err := rand.Read(idBytes); err != nil {
+		b.Log.Warn(err.Error())
+	}
+
+	message.Info.Id = strings.ToUpper(hex.EncodeToString(idBytes))
+	_, err := b.conn.Send(message)
+
+	return message.Info.Id, err
+}
+
+// Post an image message from the bridge to WhatsApp
+// Handle, for sure image/jpeg, image/png and image/gif MIME types
+func (b *Bwhatsapp) PostImageMessage(msg config.Message, filetype string) (string, error) {
+	fi := msg.Extra["file"][0].(config.FileInfo)
+
+	// Post image message
+	message := whatsapp.ImageMessage{
+		Info: whatsapp.MessageInfo{
+			RemoteJid: msg.Channel,
+		},
+		Type:    filetype,
+		Caption: msg.Username + fi.Comment,
+		Content: bytes.NewReader(*fi.Data),
+	}
+
+	b.Log.Debugf("=> Sending %#v", msg)
+
+	// create message ID
+	// TODO follow and act if https://github.com/Rhymen/go-whatsapp/issues/101 implemented
+	idBytes := make([]byte, 10)
+	if _, err := rand.Read(idBytes); err != nil {
+		b.Log.Warn(err.Error())
+	}
+
+	message.Info.Id = strings.ToUpper(hex.EncodeToString(idBytes))
+	_, err := b.conn.Send(message)
+
+	return message.Info.Id, err
+}
+
+// Send a message from the bridge to WhatsApp
+// Required implementation of the Bridger interface
+// https://github.com/42wim/matterbridge/blob/2cfd880cdb0df29771bf8f31df8d990ab897889d/bridge/bridge.go#L11-L16
+func (b *Bwhatsapp) Send(msg config.Message) (string, error) {
+	b.Log.Debugf("=> Receiving %#v", msg)
+
+	// Delete message
+	if msg.Event == config.EventMsgDelete {
+		if msg.ID == "" {
+			// No message ID in case action is executed on a message sent before the bridge was started
+			// and then the bridge cache doesn't have this message ID mapped
+
+			// TODO 42wim Doesn't the app get clogged with a ton of IDs after some time of running?
+			// WhatsApp allows to set any ID so in that case we could use external IDs and don't do mapping
+			// but external IDs are not set
+			return "", nil
+		}
+		// TODO delete message on WhatsApp https://github.com/Rhymen/go-whatsapp/issues/100
+		return "", nil
+	}
+
+	// Edit message
+	if msg.ID != "" {
+		b.Log.Debugf("updating message with id %s", msg.ID)
+
+		msg.Text += " (edited)"
+		// TODO handle edit as a message reply with updated text
+	}
+
+	// Handle Upload a file
+	if msg.Extra["file"] != nil {
+		fi := msg.Extra["file"][0].(config.FileInfo)
+		filetype := mime.TypeByExtension(filepath.Ext(fi.Name))
+
+		b.Log.Debugf("Extra file is %#v", filetype)
+
+		// TODO: add different types
+		// TODO: add webp conversion
+		switch filetype {
+		case "image/jpeg", "image/png", "image/gif":
+			return b.PostImageMessage(msg, filetype)
+		default:
+			return b.PostDocumentMessage(msg, filetype)
+		}
+	}
+
+	// Post text message
+	message := whatsapp.TextMessage{
+		Info: whatsapp.MessageInfo{
+			RemoteJid: msg.Channel, // which equals to group id
+		},
+		Text: msg.Username + msg.Text,
+	}
+
+	b.Log.Debugf("=> Sending %#v", msg)
+
+	// create message ID
+	// TODO follow and act if https://github.com/Rhymen/go-whatsapp/issues/101 implemented
+	idBytes := make([]byte, 10)
+	if _, err := rand.Read(idBytes); err != nil {
+		b.Log.Warn(err.Error())
+	}
+	message.Info.Id = strings.ToUpper(hex.EncodeToString(idBytes))
+	_, err := b.conn.Send(message)
+
+	return message.Info.Id, err
+}
+
+// TODO do we want that? to allow login with QR code from a bridged channel? https://github.com/tulir/mautrix-whatsapp/blob/513eb18e2d59bada0dd515ee1abaaf38a3bfe3d5/commands.go#L76
+//func (b *Bwhatsapp) Command(cmd string) string {
+//	return ""
+//}

bridge/xmpp/handler.go

@@ -0,0 +1,34 @@
+package bxmpp
+
+import (
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/42wim/matterbridge/bridge/helper"
+	"github.com/matterbridge/go-xmpp"
+)
+
+// handleDownloadAvatar downloads the avatar of userid from channel
+// sends a EVENT_AVATAR_DOWNLOAD message to the gateway if successful.
+// logs an error message if it fails
+func (b *Bxmpp) handleDownloadAvatar(avatar xmpp.AvatarData) {
+	rmsg := config.Message{
+		Username: "system",
+		Text:     "avatar",
+		Channel:  b.parseChannel(avatar.From),
+		Account:  b.Account,
+		UserID:   avatar.From,
+		Event:    config.EventAvatarDownload,
+		Extra:    make(map[string][]interface{}),
+	}
+	if _, ok := b.avatarMap[avatar.From]; !ok {
+		b.Log.Debugf("Avatar.From: %s", avatar.From)
+
+		err := helper.HandleDownloadSize(b.Log, &rmsg, avatar.From+".png", int64(len(avatar.Data)), b.General)
+		if err != nil {
+			b.Log.Error(err)
+			return
+		}
+		helper.HandleDownloadData(b.Log, &rmsg, avatar.From+".png", rmsg.Text, "", &avatar.Data, b.General)
+		b.Log.Debugf("Avatar download complete")
+		b.Remote <- rmsg
+	}
+}

bridge/xmpp/helpers.go

@@ -0,0 +1,30 @@
+package bxmpp
+
+import (
+	"regexp"
+
+	"github.com/42wim/matterbridge/bridge/config"
+)
+
+var pathRegex = regexp.MustCompile("[^a-zA-Z0-9]+")
+
+// GetAvatar constructs a URL for a given user-avatar if it is available in the cache.
+func getAvatar(av map[string]string, userid string, general *config.Protocol) string {
+	if hash, ok := av[userid]; ok {
+		// NOTE: This does not happen in bridge/helper/helper.go but messes up XMPP
+		id := pathRegex.ReplaceAllString(userid, "_")
+		return general.MediaServerDownload + "/" + hash + "/" + id + ".png"
+	}
+	return ""
+}
+
+func (b *Bxmpp) cacheAvatar(msg *config.Message) string {
+	fi := msg.Extra["file"][0].(config.FileInfo)
+	/* if we have a sha we have successfully uploaded the file to the media server,
+	so we can now cache the sha */
+	if fi.SHA != "" {
+		b.Log.Debugf("Added %s to %s in avatarMap", fi.SHA, msg.UserID)
+		b.avatarMap[msg.UserID] = fi.SHA
+	}
+	return ""
+}

bridge/xmpp/xmpp.go

@@ -2,7 +2,9 @@ package bxmpp
 
 import (
 	"crypto/tls"
+	"fmt"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/42wim/matterbridge/bridge"
@@ -14,49 +16,36 @@ import (
 )
 
 type Bxmpp struct {
-	xc      *xmpp.Client
-	xmppMap map[string]string
 	*bridge.Config
+
+	startTime time.Time
+	xc        *xmpp.Client
+	xmppMap   map[string]string
+	connected bool
+	sync.RWMutex
+
+	avatarAvailability map[string]bool
+	avatarMap          map[string]string
 }
 
 func New(cfg *bridge.Config) bridge.Bridger {
-	b := &Bxmpp{Config: cfg}
+	return &Bxmpp{
-	b.xmppMap = make(map[string]string)
+		Config:             cfg,
-	return b
+		xmppMap:            make(map[string]string),
+		avatarAvailability: make(map[string]bool),
+		avatarMap:          make(map[string]string),
+	}
 }
 
 func (b *Bxmpp) Connect() error {
-	var err error
 	b.Log.Infof("Connecting %s", b.GetString("Server"))
-	b.xc, err = b.createXMPP()
+	if err := b.createXMPP(); err != nil {
-	if err != nil {
 		b.Log.Debugf("%#v", err)
 		return err
 	}
+
 	b.Log.Info("Connection succeeded")
-	go func() {
+	go b.manageConnection()
-		initial := true
-		bf := &backoff.Backoff{
-			Min:    time.Second,
-			Max:    5 * time.Minute,
-			Jitter: true,
-		}
-		for {
-			if initial {
-				b.handleXMPP()
-				initial = false
-			}
-			d := bf.Duration()
-			b.Log.Infof("Disconnected. Reconnecting in %s", d)
-			time.Sleep(d)
-			b.xc, err = b.createXMPP()
-			if err == nil {
-				b.Remote <- config.Message{Username: "system", Text: "rejoin", Channel: "", Account: b.Account, Event: config.EVENT_REJOIN_CHANNELS}
-				b.handleXMPP()
-				bf.Reset()
-			}
-		}
-	}()
 	return nil
 }
 
@@ -75,41 +64,75 @@ func (b *Bxmpp) JoinChannel(channel config.ChannelInfo) error {
 }
 
 func (b *Bxmpp) Send(msg config.Message) (string, error) {
-	var msgid = ""
+	// should be fixed by using a cache instead of dropping
-	var msgreplaceid = ""
+	if !b.Connected() {
+		return "", fmt.Errorf("bridge %s not connected, dropping message %#v to bridge", b.Account, msg)
+	}
 	// ignore delete messages
-	if msg.Event == config.EVENT_MSG_DELETE {
+	if msg.Event == config.EventMsgDelete {
 		return "", nil
 	}
+
 	b.Log.Debugf("=> Receiving %#v", msg)
 
-	// Upload a file (in xmpp case send the upload URL because xmpp has no native upload support)
+	if msg.Event == config.EventAvatarDownload {
+		return b.cacheAvatar(&msg), nil
+	}
+
+	// Make a action /me of the message, prepend the username with it.
+	// https://xmpp.org/extensions/xep-0245.html
+	if msg.Event == config.EventUserAction {
+		msg.Username = "/me " + msg.Username
+	}
+
+	// Upload a file (in XMPP case send the upload URL because XMPP has no native upload support).
 	if msg.Extra != nil {
 		for _, rmsg := range helper.HandleExtra(&msg, b.General) {
-			b.xc.Send(xmpp.Chat{Type: "groupchat", Remote: rmsg.Channel + "@" + b.GetString("Muc"), Text: rmsg.Username + rmsg.Text})
+			b.Log.Debugf("=> Sending attachement message %#v", rmsg)
+			if _, err := b.xc.Send(xmpp.Chat{
+				Type:   "groupchat",
+				Remote: rmsg.Channel + "@" + b.GetString("Muc"),
+				Text:   rmsg.Username + rmsg.Text,
+			}); err != nil {
+				b.Log.WithError(err).Error("Unable to send message with share URL.")
+			}
 		}
 		if len(msg.Extra["file"]) > 0 {
-			return b.handleUploadFile(&msg)
+			return "", b.handleUploadFile(&msg)
 		}
 	}
 
-	msgid = xid.New().String()
+	var msgReplaceID string
+	msgID := xid.New().String()
 	if msg.ID != "" {
-		msgid = msg.ID
+		msgID = msg.ID
-		msgreplaceid = msg.ID
+		msgReplaceID = msg.ID
 	}
-	// Post normal message
+	// Post normal message.
-	_, err := b.xc.Send(xmpp.Chat{Type: "groupchat", Remote: msg.Channel + "@" + b.GetString("Muc"), Text: msg.Username + msg.Text, ID: msgid, ReplaceID: msgreplaceid})
+	b.Log.Debugf("=> Sending message %#v", msg)
-	if err != nil {
+	if _, err := b.xc.Send(xmpp.Chat{
+		Type:      "groupchat",
+		Remote:    msg.Channel + "@" + b.GetString("Muc"),
+		Text:      msg.Username + msg.Text,
+		ID:        msgID,
+		ReplaceID: msgReplaceID,
+	}); err != nil {
 		return "", err
 	}
-	return msgid, nil
+	return msgID, nil
 }
 
-func (b *Bxmpp) createXMPP() (*xmpp.Client, error) {
+func (b *Bxmpp) createXMPP() error {
-	tc := new(tls.Config)
+	if !strings.Contains(b.GetString("Jid"), "@") {
-	tc.InsecureSkipVerify = b.GetBool("SkipTLSVerify")
+		return fmt.Errorf("the Jid %s doesn't contain an @", b.GetString("Jid"))
-	tc.ServerName = strings.Split(b.GetString("Server"), ":")[0]
+	}
+	tc := &tls.Config{
+		ServerName:         strings.Split(b.GetString("Jid"), "@")[1],
+		InsecureSkipVerify: b.GetBool("SkipTLSVerify"), // nolint: gosec
+	}
+
+	xmpp.DebugWriter = b.Log.Writer()
+
 	options := xmpp.Options{
 		Host:                         b.GetString("Server"),
 		User:                         b.GetString("Jid"),
@@ -118,7 +141,6 @@ func (b *Bxmpp) createXMPP() (*xmpp.Client, error) {
 		StartTLS:                     true,
 		TLSConfig:                    tc,
 		Debug:                        b.GetBool("debug"),
-		Logger:                       b.Log.Writer(),
 		Session:                      true,
 		Status:                       "",
 		StatusMessage:                "",
@@ -127,7 +149,54 @@ func (b *Bxmpp) createXMPP() (*xmpp.Client, error) {
 	}
 	var err error
 	b.xc, err = options.NewClient()
-	return b.xc, err
+	return err
+}
+
+func (b *Bxmpp) manageConnection() {
+	b.setConnected(true)
+	initial := true
+	bf := &backoff.Backoff{
+		Min:    time.Second,
+		Max:    5 * time.Minute,
+		Jitter: true,
+	}
+
+	// Main connection loop. Each iteration corresponds to a successful
+	// connection attempt and the subsequent handling of the connection.
+	for {
+		if initial {
+			initial = false
+		} else {
+			b.Remote <- config.Message{
+				Username: "system",
+				Text:     "rejoin",
+				Channel:  "",
+				Account:  b.Account,
+				Event:    config.EventRejoinChannels,
+			}
+		}
+
+		if err := b.handleXMPP(); err != nil {
+			b.Log.WithError(err).Error("Disconnected.")
+			b.setConnected(false)
+		}
+
+		// Reconnection loop using an exponential back-off strategy. We
+		// only break out of the loop if we have successfully reconnected.
+		for {
+			d := bf.Duration()
+			b.Log.Infof("Reconnecting in %s.", d)
+			time.Sleep(d)
+
+			b.Log.Infof("Reconnecting now.")
+			if err := b.createXMPP(); err == nil {
+				b.setConnected(true)
+				bf.Reset()
+				break
+			}
+			b.Log.Warn("Failed to reconnect.")
+		}
+	}
 }
 
 func (b *Bxmpp) xmppKeepAlive() chan bool {
@@ -139,8 +208,7 @@ func (b *Bxmpp) xmppKeepAlive() chan bool {
 			select {
 			case <-ticker.C:
 				b.Log.Debugf("PING")
-				err := b.xc.PingC2S("", "")
+				if err := b.xc.PingC2S("", ""); err != nil {
-				if err != nil {
 					b.Log.Debugf("PING failed %#v", err)
 				}
 			case <-done:
@@ -152,40 +220,74 @@ func (b *Bxmpp) xmppKeepAlive() chan bool {
 }
 
 func (b *Bxmpp) handleXMPP() error {
-	var ok bool
+	b.startTime = time.Now()
-	var msgid string
+
 	done := b.xmppKeepAlive()
 	defer close(done)
+
 	for {
 		m, err := b.xc.Recv()
 		if err != nil {
 			return err
 		}
+
 		switch v := m.(type) {
 		case xmpp.Chat:
 			if v.Type == "groupchat" {
 				b.Log.Debugf("== Receiving %#v", v)
-				// skip invalid messages
+
+				// Skip invalid messages.
 				if b.skipMessage(v) {
 					continue
 				}
-				msgid = v.ID
-				if v.ReplaceID != "" {
-					msgid = v.ReplaceID
-				}
-				rmsg := config.Message{Username: b.parseNick(v.Remote), Text: v.Text, Channel: b.parseChannel(v.Remote), Account: b.Account, UserID: v.Remote, ID: msgid}
 
-				// check if we have an action event
+				var event string
+				if strings.Contains(v.Text, "has set the subject to:") {
+					event = config.EventTopicChange
+				}
+
+				available, sok := b.avatarAvailability[v.Remote]
+				avatar := ""
+				if !sok {
+					b.Log.Debugf("Requesting avatar data")
+					b.avatarAvailability[v.Remote] = false
+					b.xc.AvatarRequestData(v.Remote)
+				} else if available {
+					avatar = getAvatar(b.avatarMap, v.Remote, b.General)
+				}
+
+				msgID := v.ID
+				if v.ReplaceID != "" {
+					msgID = v.ReplaceID
+				}
+				rmsg := config.Message{
+					Username: b.parseNick(v.Remote),
+					Text:     v.Text,
+					Channel:  b.parseChannel(v.Remote),
+					Account:  b.Account,
+					Avatar:   avatar,
+					UserID:   v.Remote,
+					ID:       msgID,
+					Event:    event,
+				}
+
+				// Check if we have an action event.
+				var ok bool
 				rmsg.Text, ok = b.replaceAction(rmsg.Text)
 				if ok {
-					rmsg.Event = config.EVENT_USER_ACTION
+					rmsg.Event = config.EventUserAction
 				}
+
 				b.Log.Debugf("<= Sending message from %s on %s to gateway", rmsg.Username, b.Account)
 				b.Log.Debugf("<= Message is %#v", rmsg)
 				b.Remote <- rmsg
 			}
+		case xmpp.AvatarData:
+			b.handleDownloadAvatar(v)
+			b.avatarAvailability[v.From] = true
+			b.Log.Debugf("Avatar for %s is now available", v.From)
 		case xmpp.Presence:
-			// do nothing
+			// Do nothing.
 		}
 	}
 }
@@ -198,30 +300,41 @@ func (b *Bxmpp) replaceAction(text string) (string, bool) {
 }
 
 // handleUploadFile handles native upload of files
-func (b *Bxmpp) handleUploadFile(msg *config.Message) (string, error) {
+func (b *Bxmpp) handleUploadFile(msg *config.Message) error {
-	var urldesc = ""
+	var urlDesc string
 
-	for _, f := range msg.Extra["file"] {
+	for _, file := range msg.Extra["file"] {
-		fi := f.(config.FileInfo)
+		fileInfo := file.(config.FileInfo)
-		if fi.Comment != "" {
+		if fileInfo.Comment != "" {
-			msg.Text += fi.Comment + ": "
+			msg.Text += fileInfo.Comment + ": "
 		}
-		if fi.URL != "" {
+		if fileInfo.URL != "" {
-			msg.Text = fi.URL
+			msg.Text = fileInfo.URL
-			if fi.Comment != "" {
+			if fileInfo.Comment != "" {
-				msg.Text = fi.Comment + ": " + fi.URL
+				msg.Text = fileInfo.Comment + ": " + fileInfo.URL
-				urldesc = fi.Comment
+				urlDesc = fileInfo.Comment
 			}
 		}
-		_, err := b.xc.Send(xmpp.Chat{Type: "groupchat", Remote: msg.Channel + "@" + b.GetString("Muc"), Text: msg.Username + msg.Text})
+		if _, err := b.xc.Send(xmpp.Chat{
-		if err != nil {
+			Type:   "groupchat",
-			return "", err
+			Remote: msg.Channel + "@" + b.GetString("Muc"),
+			Text:   msg.Username + msg.Text,
+		}); err != nil {
+			return err
 		}
-		if fi.URL != "" {
+
-			b.xc.SendOOB(xmpp.Chat{Type: "groupchat", Remote: msg.Channel + "@" + b.GetString("Muc"), Ooburl: fi.URL, Oobdesc: urldesc})
+		if fileInfo.URL != "" {
+			if _, err := b.xc.SendOOB(xmpp.Chat{
+				Type:    "groupchat",
+				Remote:  msg.Channel + "@" + b.GetString("Muc"),
+				Ooburl:  fileInfo.URL,
+				Oobdesc: urlDesc,
+			}); err != nil {
+				b.Log.WithError(err).Warn("Failed to send share URL.")
+			}
 		}
 	}
-	return "", nil
+	return nil
 }
 
 func (b *Bxmpp) parseNick(remote string) string {
@@ -260,7 +373,23 @@ func (b *Bxmpp) skipMessage(message xmpp.Chat) bool {
 		return true
 	}
 
+	// do not show subjects on connect #732
+	if strings.Contains(message.Text, "has set the subject to:") && time.Since(b.startTime) < time.Second*5 {
+		return true
+	}
+
 	// skip delayed messages
-	t := time.Time{}
+	return !message.Stamp.IsZero() && time.Since(message.Stamp).Minutes() > 5
-	return message.Stamp != t
+}
+
+func (b *Bxmpp) setConnected(state bool) {
+	b.Lock()
+	b.connected = state
+	defer b.Unlock()
+}
+
+func (b *Bxmpp) Connected() bool {
+	b.RLock()
+	defer b.RUnlock()
+	return b.connected
 }

bridge/zulip/zulip.go

@@ -4,6 +4,8 @@ import (
 	"encoding/json"
 	"io/ioutil"
 	"strconv"
+	"strings"
+	"sync"
 	"time"
 
 	"github.com/42wim/matterbridge/bridge"
@@ -17,6 +19,7 @@ type Bzulip struct {
 	bot     *gzb.Bot
 	streams map[int]string
 	*bridge.Config
+	sync.RWMutex
 }
 
 func New(cfg *bridge.Config) bridge.Bridger {
@@ -52,7 +55,7 @@ func (b *Bzulip) Send(msg config.Message) (string, error) {
 	b.Log.Debugf("=> Receiving %#v", msg)
 
 	// Delete message
-	if msg.Event == config.EVENT_MSG_DELETE {
+	if msg.Event == config.EventMsgDelete {
 		if msg.ID == "" {
 			return "", nil
 		}
@@ -100,27 +103,67 @@ func (b *Bzulip) getChannel(id int) string {
 
 func (b *Bzulip) handleQueue() error {
 	for {
-		messages, _ := b.q.GetEvents()
+		messages, err := b.q.GetEvents()
+		switch err {
+		case gzb.BackoffError:
+			time.Sleep(time.Second * 5)
+		case gzb.NoJSONError:
+			b.Log.Error("Response wasn't JSON, server down or restarting? sleeping 10 seconds")
+			time.Sleep(time.Second * 10)
+		case gzb.BadEventQueueError:
+			b.Log.Info("got a bad event queue id error, reconnecting")
+			b.bot.Queues = nil
+			for {
+				b.q, err = b.bot.RegisterAll()
+				if err != nil {
+					b.Log.Errorf("reconnecting failed: %s. Sleeping 10 seconds", err)
+					time.Sleep(time.Second * 10)
+				}
+				break
+			}
+		case gzb.HeartbeatError:
+			b.Log.Debug("heartbeat received.")
+		default:
+			b.Log.Debugf("receiving error: %#v", err)
+		}
+		if err != nil {
+			continue
+		}
 		for _, m := range messages {
 			b.Log.Debugf("== Receiving %#v", m)
 			// ignore our own messages
 			if m.SenderEmail == b.GetString("login") {
 				continue
 			}
-			rmsg := config.Message{Username: m.SenderFullName, Text: m.Content, Channel: b.getChannel(m.StreamID), Account: b.Account, UserID: strconv.Itoa(m.SenderID), Avatar: m.AvatarURL}
+
+			avatarURL := m.AvatarURL
+			if !strings.HasPrefix(avatarURL, "http") {
+				avatarURL = b.GetString("server") + avatarURL
+			}
+
+			rmsg := config.Message{
+				Username: m.SenderFullName,
+				Text:     m.Content,
+				Channel:  b.getChannel(m.StreamID) + "/topic:" + m.Subject,
+				Account:  b.Account,
+				UserID:   strconv.Itoa(m.SenderID),
+				Avatar:   avatarURL,
+			}
 			b.Log.Debugf("<= Sending message from %s on %s to gateway", rmsg.Username, b.Account)
 			b.Log.Debugf("<= Message is %#v", rmsg)
 			b.Remote <- rmsg
-			b.q.LastEventID = m.ID
 		}
+
 		time.Sleep(time.Second * 3)
 	}
 }
 
 func (b *Bzulip) sendMessage(msg config.Message) (string, error) {
-	topic := "matterbridge"
+	topic := ""
-	if b.GetString("topic") != "" {
+	if strings.Contains(msg.Channel, "/topic:") {
-		topic = b.GetString("topic")
+		res := strings.Split(msg.Channel, "/topic:")
+		topic = res[1]
+		msg.Channel = res[0]
 	}
 	m := gzb.Message{
 		Stream:  msg.Channel,

changelog.md

@@ -1,3 +1,547 @@
+# v1.18.3
+
+## Enhancements
+
+- nctalk: Add TLSConfig to nctalk (#1195)
+- whatsapp: Handle broadcasts as groups in Whatsapp #1213
+- matrix: switch to upstream gomatrix #1219
+- api: support multiple websocket clients #1205
+
+## Bugfix
+
+- general: update vendor
+- zulip: Check location of avatarURL (zulip). Fixes #1214 (#1227)
+- nctalk: Fix issue with too many open files #1223
+- nctalk: Fix mentions #1222
+- nctalk: Fix message replays #1220
+
+This release couldn't exist without the following contributors:
+@gary-kim, @tilosp, @NikkyAI, @escoand, @42wim
+
+# v1.18.2
+
+## Bugfix
+
+- zulip: Fix error loop (zulip) (#1210)
+- whatsapp: Update whatsapp vendor and fix a panic (#1209)
+
+This release couldn't exist without the following contributors:
+@SuperSandro2000, @42wim
+
+# v1.18.1
+
+## New features
+
+- telegram: Support Telegram animated stickers (tgs) format (#1173). See https://github.com/42wim/matterbridge/wiki/Settings#mediaConverttgs for more info
+
+## Enhancements
+
+- matrix: Remove HTML formatting for push messages (#1188) (#1189)
+- mattermost: Use mattermost v5 module (#1192)
+
+## Bugfix
+
+- whatsapp: Handle panic in whatsapp. Fixes #1180 (#1184)
+- nctalk: Fix Nextcloud Talk connection failure (#1179)
+- matrix: Sleep when ratelimited on joins (matrix). Fixes #1201 (#1206)
+
+This release couldn't exist without the following contributors:
+@42wim, @BenWiederhake, @Dellle, @gary-kim
+
+# v1.18.0
+
+## New features
+
+- nctalk: new protocol added. Add Nextcloud Talk support #1167
+- general: Add an option to log into a file rather than stdout (#1168)
+- api: Add websocket to API (#970)
+
+## Enhancements
+
+- telegram: Fix MarkdownV2 support in Telegram (#1169)
+- whatsapp: Reload user information when a new contact is detected (whatsapp) (#1160)
+- api: Add sane RemoteNickFormat default for API (#1157)
+- irc: Skip gIRC built-in rate limiting (irc) (#1164)
+- irc: Only colour IRC nicks if there is one. (#1161)
+- docker: Combine runs to one layer (#1151)
+
+## Bugfix
+
+- general: Update dependencies for 1.18.0 release (#1175)
+
+Discord users are encouraged to upgrade, this release works with the move to the discord.com domain.
+
+This release couldn't exist without the following contributors:
+@42wim, @jlu5, @qaisjp, @TheHolyRoger, @SuperSandro2000, @gary-kim, @z3bra, @greenx, @haykam821, @nathanaelhoun
+
+# v1.17.5
+
+## Enhancements
+
+- irc: Add StripMarkdown option (irc). (#1145)
+- general: Increase debug logging with function,file and linenumber (#1147)
+- general: Update Dockerfile so inotify works (#1148)
+- matrix: Add an option to disable sending HTML to matrix. Fixes #1022 (#1135)
+- xmpp: Implement xep-0245 (xmpp). Closes #1137 (#1144)
+
+## Bugfix
+
+- discord: Fix #1120: replaceAction "_" crash (discord) (#1121)
+- discord: Fix #1049: missing space before embeds (discord) (#1124)
+- discord: Fix webhook EventUserAction messages being skipped (discord) (#1133)
+- matrix: Avoid creating invalid url when the user doesn't have an avatar (matrix) (#1130)
+- msteams: Ignore non-user messages (msteams). Fixes #1141 (#1149)
+- slack: Do not use webhooks when token is configured (slack) (fixes #1123) (#1134)
+- telegram: Fix forward from hidden users (telegram). Closes #1131 (#1143)
+- xmpp: Prevent re-requesting avatar data (xmpp) (#1117)
+
+This release couldn't exist without the following contributors:
+@qaisjp, @xnaas, @42wim, @Polynomdivision, @tfve
+
+# v1.17.4
+
+## Bugfix
+
+- general: Lowercase account names. Fixes #1108 (#1110)
+- msteams: Remove panics and retry polling on failure (msteams). Fixes #1104 (#1105
+- whatsapp: Update Rhymen/go-whatsapp. Fixes #1107 (#1109) (make whatsapp working again)
+- discord: Add an ID cache (discord). Fixes #1106 (#1111) (fix delete/edits with webhooks)
+
+# v1.17.3
+
+## Enhancements
+
+- xmpp: Implement User Avatar spoofing of XMPP users #1090
+- rocketchat: Relay Joins/Topic changes in RocketChat bridge (#1085)
+- irc: Add JoinDelay option (irc). Fixes #1084 (#1098)
+- slack: Clip too long messages on 3000 length (slack). Fixes #1081 (#1102)
+
+## Bugfix
+
+- general: Fix the behavior of ShowTopicChange and SyncTopic (#1086)
+- slack: Prevent image/message looping (slack). Fixes #1088 (#1096)
+- whatsapp: Ignore non-critical errors (whatsapp). Fixes #1094 (#1100)
+- irc: Add extra space before colon in attachments (irc). Fixes #1089 (#1101)
+
+This release couldn't exist without the following contributors:
+@42wim, @ldruschk, @qaisjp, @Polynomdivision
+
+# v1.17.2
+
+## Enhancements
+
+- slack: Update vendor slack-go/slack (#1068)
+- general: Update vendor d5/tengo (#1066)
+- general: Clarify terminology used in mapping group chat IDs to channels in config (#1079)
+
+## Bugfix
+
+- whatsapp: Update Rhymen/go-whatsapp vendor and whatsapp version (#1078). Fixes Media upload #1074
+- whatsapp: Reset start timestamp on reconnect (whatsapp). Fixes #1059 (#1064)
+
+This release couldn't exist without the following contributors:
+@42wim, @jheiselman
+
+# v1.17.1
+
+## Enhancements
+
+- docker: Remove build dependencies from final image (multistage build) #1057
+
+## Bugfix
+
+- general: Don't transmit typing events from ourselves #1056
+- general: Add support for build tags #1054
+- discord: Strip extra info from emotes (discord) #1052
+- msteams: fix macos build: Update vendor yaegashi/msgraph.go to v0.1.2 #1036
+- whatsapp: Update client version whatsapp. Fixes #1061 #1062
+
+This release couldn't exist without the following contributors:
+@awigen, @qaisjp, @42wim
+
+# v1.17.0
+
+## New features
+
+- msteams: new protocol added. Add initial Microsoft Teams support #967
+  See https://github.com/42wim/matterbridge/wiki/MS-Teams-setup for a complete walkthrough
+- discord: Add ability to procure avatars from the destination bridge #1000
+- matrix: Add support for avatars from matrix. #1007
+- general: support JSON and YAML config formats #1045
+
+## Enhancements
+
+- discord: Check only bridged channels for PermManageWebhooks #1001
+- irc: Be less lossy when throttling IRC messages #1004
+- keybase: updated library #1002, #1019
+- matrix: Rebase gomatrix vendor with upstream #1006
+- slack: Use upstream slack-go/slack again #1018
+- slack: Ignore ConnectingEvent #1041
+- slack: use blocks not attachments #1048
+- sshchat: Update vendor shazow/ssh-chat #1029
+- telegram: added markdownv2 mode for telegram #1037
+- whatsapp: Implement basic reconnect (whatsapp). Fixes #987 #1003
+
+## Bugfix
+
+- discord: Fix webhook permission checks sometimes failing #1043
+- discord: Fix #1027: warning when handling inbound webhooks #1044
+- discord: Fix duplicate separator on empty description/url (discord) #1035
+- matrix: Fix issue with underscores in links #999
+- slack: Fix #1039: messages sent to Slack being synced back #1046
+- telegram: Make avatars download work with mediaserverdownload (telegram). Fixes #920 
+
+This release couldn't exist without the following contributors:
+@qaisjp, @jakubgs, @burner1024, @notpushkin, @MartijnBraam, @42wim
+
+# v1.16.5
+
+- Fix version bump
+
+# v1.16.4
+
+## New features
+
+- whatsapp: Add support for WhatsApp media (jpeg/png/gif) bridging (#974)
+- telegram: Add QuoteLengthLimit option (telegram) fixes #963 (#985)
+- telegram: Add DisableWebPagePreview option (telegram). Closes #980 (#994)
+
+## Enhancements
+
+- general: update dependencies
+- tengo: update to tengo v2
+- general: Add Docker Compose configuration (#990)
+
+## Bugfix
+
+- general: Fail with message instead of panic. #988 (#991)
+- telegram: Add extra mimetypes to docker image. Fixes #969
+- discord: Fix channel ID problem with multiple gateways (discord). Fixes #953 (#977)
+- discord: Show file comment in webhook if normal message is empty (discord). Fixes #962 (#995)
+- matrix: Fix parsing issues - Disable smartypants in markdown parser. Fixes #989, #983 (#993)
+- sshchat: Fix duplicated messages (sshchat). Fixes #950 (#996)
+
+This release couldn't exist without the following contributors:
+@jwflory, @42wim, @pbek, @Humorhenker, @c0ncord2, @glazzara
+
+# v1.16.3
+
+## Bugfix
+
+- slack: Fix issues with ratelimiting #959
+- mattermost: Fix bug when using webhookURL and login/token together #960
+
+# v1.16.2
+
+## New features
+
+- keybase: Add support for receiving attachments (keybase) (#923)
+
+## Enhancements
+
+- general: Switch to new emoji library kyokomi/emoji (#948)
+- general: Update markdown parsing library to github.com/gomarkdown/markdown (#944)
+- ssh-chat: Update shazow/ssh-chat dependency (#947)
+
+## Bugfix
+
+- slack: Fix issues with the slack block kit API #937 (#943).
+
+This release couldn't exist without the following contributors:
+@42wim, @bmpickford, @goncalor
+
+# v1.16.1
+
+## New features
+
+* rocketchat: add token support #892
+* matrix: Add support for uploading application/x and audio/x (matrix). #929
+
+## Enhancements
+
+* general: Do configuration validation on start-up. Fixes #888
+* general: updated vendored libraries (discord/whatsapp) #932
+* discord: user typing messages #914
+* slack: Convert slack bold/strike to correct markdown (slack). Fixes #918
+
+## Bugfix
+
+* discord: fix Failed to fetch information for members message. #894
+* discord: remove obsolete file upload links (discord). #931
+* slack: suppress unhandled HelloEvent message #913
+* mattermost: Fix panic on WebhookURL only setting (mattermost). #917
+* matrix: fix corrupted links between slack and matrix #924
+
+This release couldn't exist without the following contributors:
+@qaisjp, @hramrach, @42wim
+
+# v1.16.0
+
+## New features
+
+* keybase: new protocol added. Add initial Keybase Chat support #877 Thanks to @hyperobject
+* discord: Support webhook files in discord #872
+
+## Enhancements
+
+* general: update dependencies
+
+## Bugfix
+
+* discord: Underscores from Discord don't arrive correctly #864
+* xmpp: Fix possible panic at startup of the XMPP bridge #869
+* mattermost: Make getChannelIdTeam behave like GetChannelId for groups (mattermost) #873
+
+This release couldn't exist without the following contributors:
+@hyperobject, @42wim, @bucko909, @MOZGIII
+
+# v1.15.1
+
+## New features
+* discord: Support webhook message deletions (discord) (#853)
+
+## Enhancements
+
+* discord: Support bulk deletions #851
+* discord: Support channels in categories #863 (use category/channel. See matterbridge.toml.sample for more info)
+* mattermost: Add an option to skip the Mattermost server version check #849
+
+## Bugfix
+
+* xmpp: fix segfault when disconnected/reconnected #856
+* telegram: fix panic in handleEntities #858
+
+This release couldn't exist without the following contributors:
+@42wim, @qaisjp, @joohoi
+
+# v1.15.0
+## New features
+* Add scripting (tengo) support for every outgoing message (#806)
+  See https://github.com/42wim/matterbridge/wiki/Settings#tengo and 
+  https://github.com/42wim/matterbridge/wiki/Settings#outmessage for more information
+* Add tengo support to RemoteNickFormat (#793)
+  See https://github.com/42wim/matterbridge/wiki/Settings#remotenickformat-2
+  * Deprecated `Message` under `[tengo]` to `InMessage`
+
+## Enhancements
+  * general: Forward only user-typing messages if supported by protocol (#832)
+  * general: updated wiki with all possible settings: https://github.com/42wim/matterbridge/wiki/Settings
+  * tengo: Add msg event to tengo
+  * xmpp: Verify TLS against JID domain, not the host. (xmpp) (#834)
+  * xmpp: Allow messages with timestamp (xmpp). Fixes #835 (#847)
+  * irc: Add verbose IRC joins/parts (ident@host) (#805)
+  See https://github.com/42wim/matterbridge/wiki/Settings#verbosejoinpart
+  * rocketchat: Add useraction support (rocketchat). Closes #772 (#794)
+
+## Bugfix
+  * slack: Fix regression in autojoining with legacy tokens (slack). Fixes #651 (#848)
+  * xmpp: Revert xmpp to orig behaviour. Closes #844
+* whatsapp: Update github.com/Rhymen/go-whatsapp vendor. Fixes #843
+* mattermost: Update channels of all teams (mattermost)
+
+This release couldn't exist without the following contributors:
+@42wim, @Helcaraxan, @chotaire, @qaisjp, @dajohi, @kousu
+
+# v1.14.4
+
+## Bugfix
+* mattermost: Add Id to EditMessage (mattermost). Fixes #802
+* mattermost: Fix panic on nil message.Post (mattermost). Fixes #804
+* mattermost: Handle unthreaded messages (mattermost). Fixes #803
+* mattermost: Use paging in initUser and UpdateUsers (mattermost)
+* slack: Add lacking clean-up in Slack synchronisation (#811)
+* slack: Disable user lookups on delete messages (slack) (#812)
+
+# v1.14.3
+
+## Bugfix
+* irc: Fix deadlock on reconnect (irc). Closes #757
+
+# v1.14.2
+
+## Bugfix
+* general: Update tengo vendor and load the stdlib. Fixes #789 (#792)
+* rocketchat: Look up #channel too (rocketchat). Fix #773 (#775)
+* slack: Ignore messagereplied and hidden messages (slack). Fixes #709 (#779)
+* telegram: Handle nil message (telegram). Fixes #777
+* irc: Use default nick if none specified (irc). Fixes #785
+* irc: Return when not connected and drop a message (irc). Fixes #786
+* irc: Revert fix for #722 (Support quits from irc correctly). Closes #781
+
+## Contributors
+This release couldn't exist without the following contributors:
+@42wim, @Helcaraxan, @dajohi
+
+# v1.14.1
+## Bugfix
+* slack: Fix crash double unlock (slack) (#771)
+
+# v1.14.0
+
+## Breaking
+* zulip: Need to specify /topic:mytopic for channel configuration (zulip). (#751)
+
+## New features
+* whatsapp: new protocol added. Add initial WhatsApp support (#711) Thanks to @KrzysztofMadejski
+* facebook messenger: new protocol via matterbridge api. See https://github.com/VictorNine/fbridge/ for more information.
+* general: Add scripting (tengo) support for every incoming message (#731). See `TengoModifyMessage`
+* general: Allow regexs in ignoreNicks. Closes #690 (#720)
+* general: Support rewriting messages from relaybots using ExtractNicks. Fixes #466 (#730). See `ExtractNicks` in matterbridge.toml.sample
+* general: refactor Make all loggers derive from non-default instance (#728). Thanks to @Helcaraxan
+* rocketchat: add support for the rocketchat API. Sending to rocketchat now supports uploading of files, editing and deleting of messages.
+* discord: Support join/leaves from discord. Closes #654 (#721)
+* discord: Allow sending discriminator with Discord username (#726). See `UseDiscriminator` in matterbridge.toml.sample
+* slack: Add extra debug option (slack). See `Debug` in the slack section in matterbridge.toml.sample
+* telegram: Add support for URL in messageEntities (telegram). Fixes #735 (#736)
+* telegram: Add MediaConvertWebPToPNG option (telegram). (#741). See `MediaConvertWebPToPNG` in matterbridge.toml.sample
+
+## Enhancements
+* general: Fail gracefully on incorrect human input. Fixes #739 (#740)
+* matrix: Detect html nicks in RemoteNickFormat (matrix). Fixes #696 (#719)
+* matrix: Send notices on join/parts (matrix). Fixes #712 (#716)
+
+## Bugfix
+* general: Handle file upload/download only once for each message (#742)
+* zulip: Fix error handling on bad event queue id (zulip). Closes #694
+* zulip: Keep reconnecting until succeed (zulip) (#737)
+* irc: add support for (older) unrealircd versions. #708
+* irc: Support quits from irc correctly. Fixes #722 (#724)
+* matrix: Send username when uploading video/images (matrix). Fixes #715 (#717)
+* matrix: Trim <p> and </p> tags (matrix). Closes #686 (#753)
+* slack: Hint at thread replies when messages are unthreaded (slack) (#684)
+* slack: Fix race-condition in populateUser() (#767)
+* xmpp: Do not send topic changes on connect (xmpp). Fixes #732 (#733)
+* telegram: Fix regression in HTML handling (telegram). Closes #734
+* discord: Do not relay any bot messages (discord) (#743)
+* rocketchat: Do not send duplicate messages (rocketchat). Fixes #745 (#752)
+
+## Contributors
+This release couldn't exist without the following contributors:
+@Helcaraxan, @KrzysztofMadejski, @AJolly, @DeclanHoare
+
+# v1.13.1
+
+This release fixes go modules issues because of https://github.com/labstack/echo/issues/1272
+
+## Bugfix
+* general: fixes Unable to build 1.13.0 #698
+* api: move to labstack/echo/v4 fixes #698
+
+# v1.13.0
+
+## New features
+* general: refactors of telegram, irc, mattermost, matrix, discord, sshchat bridges and the gateway.
+* irc: Add option to send RAW commands after connection (irc) #490. See `RunCommands` in matterbridge.toml.sample
+* mattermost: 3.x support dropped
+* mattermost: Add support for mattermost threading (#627)
+* slack: Sync channel topics between Slack bridges #585. See `SyncTopic` in matterbridge.toml.sample
+* matrix: Add support for markdown to HTML conversion (matrix). Closes #663 (#670)
+* discord: Improve error reporting on failure to join Discord. Fixes #672 (#680)
+* discord: Use only one webhook if possible (discord) (#681)
+* discord: Allow to bridge non-bot Discord users (discord) (#689) If you prefix a token with `User ` it'll treat is as a user token.
+
+## Bugfix
+* slack: Try downloading files again if slack is too slow (slack). Closes #655 (#656)
+* slack: Ignore LatencyReport event (slack)
+* slack: Fix #668 strip lang in code fences sent to Slack (#673)
+* sshchat: Fix sshchat connection logic (#661)
+* sshchat: set quiet mode to filter joins/quits
+* sshchat: Trim newlines in the end of relayed messages
+* sshchat: fix media links
+* sshchat: do not relay "Rate limiting is in effect" message
+* mattermost: Fail if channel starts with hashtag (mattermost). Closes #625
+* discord: Add file comment to webhook messages (discord). Fixes #358
+* matrix: Fix displaying usernames for plain text clients. (matrix) (#685)
+* irc: Fix possible data race (irc). Closes #693
+* irc: Handle servers without MOTD (irc). Closes #692
+
+# v1.12.3
+## Bugfix
+* slack: Fix bot (legacy token) messages not being send. Closes #571
+* slack: Populate user on channel join (slack) (#644)
+* slack: Add wait option for populateUsers/Channels (slack) Fixes #579 (#653)
+
+# v1.12.2
+
+## Bugfix
+* irc: Fix multiple channel join regression. Closes #639
+* slack: Make slack-legacy change less restrictive (#626)
+
+# v1.12.1
+
+## Bugfix
+* discord: fix regression on server ID connection #619 #617
+* discord: Limit discord username via webhook to 32 chars
+* slack: Make sure threaded files stay in thread (slack). Fixes #590
+* slack: Do not post empty messages (slack). Fixes #574
+* slack: Handle deleted/edited thread starting messages (slack). Fixes #600 (#605)
+* irc: Rework connection logic (irc)
+* irc: Fix Nickserv logic (irc) #602
+
+# v1.12.0
+
+## Breaking changes
+The slack bridge has been split in a `slack-legacy` and `slack` bridge.
+If you're still using `legacy tokens` and want to keep using them you'll have to rename `slack` to `slack-legacy` in your configuration. See [wiki](https://github.com/42wim/matterbridge/wiki/Section-Slack-(basic)#legacy-configuration) for more information.
+
+To migrate to the new bot-token based setup you can follow the instructions [here](https://github.com/42wim/matterbridge/wiki/Slack-bot-setup).
+
+Slack legacy tokens may be deprecated by Slack at short notice, so it is STRONGLY recommended to use a proper bot-token instead.
+
+## New features
+* general: New {GATEWAY} variable for `RemoteNickFormat` #501. See `RemoteNickFormat` in matterbridge.toml.sample.
+* general: New {CHANNEL} variable for `RemoteNickFormat` #515. See `RemoteNickFormat` in matterbridge.toml.sample.
+* general: Remove hyphens when auto-loading envvars from viper config #545
+* discord: You can mention discord-users from other bridges.
+* slack: Preserve threading between Slack instances #529. See `PreserveThreading` in matterbridge.toml.sample.
+* slack: Add ability to show when user is typing across Slack bridges #559
+* slack: Add rate-limiting
+* mattermost: Add support for mattermost [matterbridge plugin](https://github.com/matterbridge/mattermost-plugin)
+* api: Respond with message on connect. #550
+* api: Add a health endpoint to API #554
+
+## Bugfix
+* slack: Refactoring and making it better.
+* slack: Restore file comments coming from Slack. #583
+* irc: Fix IRC line splitting. #587
+* mattermost: Fix cookie and personal token behaviour. #530
+* mattermost: Check for expiring sessions and reconnect.
+
+
+## Contributors
+This release couldn't exist without the following contributors:
+@jheiselman, @NikkyAI, @dajohi, @NetwideRogue, @patcon and @Helcaraxan
+
+Special thanks to @Helcaraxan and @patcon for their work on improving/refactoring slack.
+
+# v1.11.3
+
+## Bugfix
+* mattermost: fix panic when using webhooks #491
+* slack: fix issues regarding API changes and lots of channels #489
+* irc: fix rejoin on kick problem #488
+
+# v1.11.2
+
+## Bugfix
+* slack: fix slack API changes regarding to files/images
+
+# v1.11.1
+
+## New features
+* slack: Add support for slack channels by ID. Closes #436
+* discord: Clip too long messages sent to discord (discord). Closes #440
+
+## Bugfix
+* general: fix possible panic on downloads that are too big #448
+* general: Fix avatar uploads to work with MediaDownloadPath. Closes #454
+* discord: allow receiving of topic changes/channel leave/joins from other bridges through the webhook
+* discord: Add a space before url in file uploads (discord). Closes #461
+* discord:  Skip empty messages being sent with the webhook (discord). #469
+* mattermost: Use nickname instead of username if defined (mattermost). Closes #452
+* irc: Stop numbers being stripped after non-color control codes (irc) (#465)
+* slack: Use UserID to look for avatar instead of username (slack). Closes #472
+
 # v1.11.0
 
 ## New features

ci/bintray.sh

@@ -1,27 +0,0 @@
-#!/bin/bash
-go version |grep go1.10 || exit
-VERSION=$(git describe --tags)
-mkdir ci/binaries
-GOOS=windows GOARCH=amd64 go build -ldflags "-s -w -X main.githash=$(git log --pretty=format:'%h' -n 1)" -o ci/binaries/matterbridge-$VERSION-windows-amd64.exe
-GOOS=linux GOARCH=amd64 go build -ldflags "-s -w -X main.githash=$(git log --pretty=format:'%h' -n 1)" -o ci/binaries/matterbridge-$VERSION-linux-amd64
-GOOS=linux GOARCH=arm go build -ldflags "-s -w -X main.githash=$(git log --pretty=format:'%h' -n 1)" -o ci/binaries/matterbridge-$VERSION-linux-arm
-GOOS=darwin GOARCH=amd64 go build -ldflags "-s -w -X main.githash=$(git log --pretty=format:'%h' -n 1)" -o ci/binaries/matterbridge-$VERSION-darwin-amd64
-cd ci
-cat > deploy.json <<EOF
-{
-    "package": {
-        "name": "Matterbridge",
-        "repo": "nightly",
-        "subject": "42wim"
-    },
-    "version": {
-        "name": "$VERSION"
-    },
-    "files":
-        [
-        {"includePattern": "ci/binaries/(.*)", "uploadPattern":"\$1"}
-        ],
-    "publish": true
-}
-EOF
-

contrib/api.yaml

@@ -0,0 +1,210 @@
+openapi: 3.0.0
+info:
+  contact: {}
+  description: A read/write API for the Matterbridge chat bridge.
+  license:
+    name: Apache 2.0
+    url: 'https://github.com/42wim/matterbridge/blob/master/LICENSE'
+  title: Matterbridge API
+  version: "0.1.0-oas3"
+paths:
+  /health:
+    get:
+      responses:
+        '200':
+          description: OK
+          content:
+            '*/*':
+              schema:
+                type: string
+      summary: Checks if the server is alive.
+  /message:
+    post:
+      responses:
+        '200':
+          description: OK
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/config.OutgoingMessageResponse'
+      summary: Create a message
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/config.OutgoingMessage'
+        description: Message object to create
+        required: true
+  /messages:
+    get:
+      responses:
+        '200':
+          description: OK
+          content:
+            application/json:
+              schema:
+                items:
+                  $ref: '#/components/schemas/config.IncomingMessage'
+                type: array
+      security:
+        - ApiKeyAuth: []
+      summary: List new messages
+  /stream:
+    get:
+      responses:
+        '200':
+          description: OK
+          content:
+            application/x-json-stream:
+              schema:
+                $ref: '#/components/schemas/config.IncomingMessage'
+      summary: Stream realtime messages
+servers:
+  - url: /api
+components:
+  securitySchemes:
+    bearerAuth:
+      type: http
+      scheme: bearer
+  schemas:
+    config.IncomingMessage:
+      properties:
+        avatar:
+          description: URL to an avatar image
+          example: >-
+            https://secure.gravatar.com/avatar/1234567890abcdef1234567890abcdef.jpg
+          type: string
+        event:
+          description: >-
+            A specific matterbridge event. (see
+            https://github.com/42wim/matterbridge/blob/master/bridge/config/config.go#L16)
+          type: string
+        gateway:
+          description: Name of the gateway as configured in matterbridge.toml
+          example: mygateway
+          type: string
+        text:
+          description: Content of the message
+          example: 'Testing, testing, 1-2-3.'
+          type: string
+        username:
+          description: Human-readable username
+          example: alice
+          type: string
+        account:
+          description: Unique account name of format "[protocol].[slug]" as defined in matterbridge.toml 
+          example: slack.myteam
+          type: string
+        channel:
+          description: Human-readable channel name of sending bridge
+          example: test-channel
+          type: string
+        id:
+          description: Unique ID of message on the gateway
+          example: slack 1541361213.030700
+          type: string
+        parent_id:
+          description: Unique ID of a parent message, if threaded
+          example: slack 1541361213.030700
+          type: string
+        protocol:
+          description: Chat protocol of the sending bridge
+          example: slack
+          type: string
+        timestamp:
+          description: Timestamp of the message
+          example: "1541361213.030700"
+          type: string
+        userid:
+          description: Userid on the sending bridge
+          example: U4MCXJKNC
+          type: string
+        extra:
+          description: Extra data that doesn't fit in other fields (eg base64 encoded files)
+          type: object
+    config.OutgoingMessage:
+      properties:
+        avatar:
+          description: URL to an avatar image
+          example: >-
+            https://secure.gravatar.com/avatar/1234567890abcdef1234567890abcdef.jpg
+          type: string
+        event:
+          description: >-
+            A specific matterbridge event. (see
+            https://github.com/42wim/matterbridge/blob/master/bridge/config/config.go#L16)
+          example: ""
+          type: string
+        gateway:
+          description: Name of the gateway as configured in matterbridge.toml
+          example: mygateway
+          type: string
+        text:
+          description: Content of the message
+          example: 'Testing, testing, 1-2-3.'
+          type: string
+        username:
+          description: Human-readable username
+          example: alice
+          type: string
+      type: object
+      required:
+        - gateway
+        - text
+        - username
+    config.OutgoingMessageResponse:
+      properties:
+        avatar:
+          description: URL to an avatar image
+          example: >-
+            https://secure.gravatar.com/avatar/1234567890abcdef1234567890abcdef.jpg
+          type: string
+        event:
+          description: >-
+            A specific matterbridge event. (see
+            https://github.com/42wim/matterbridge/blob/master/bridge/config/config.go#L16)
+          example: ""
+          type: string
+        gateway:
+          description: Name of the gateway as configured in matterbridge.toml
+          example: mygateway
+          type: string
+        text:
+          description: Content of the message
+          example: 'Testing, testing, 1-2-3.'
+          type: string
+        username:
+          description: Human-readable username
+          example: alice
+          type: string
+        account:
+          description: fixed api account 
+          example: api.local
+          type: string
+        channel:
+          description: fixed api channel 
+          example: api
+          type: string
+        id:
+          example: ""
+          type: string
+        parent_id:
+          example: ""
+          type: string
+        protocol:
+          description: fixed api protocol
+          example: api
+          type: string
+        timestamp:
+          description: Timestamp of the message
+          example: "1541361213.030700"
+          type: string
+        userid:
+          example: ""
+          type: string
+        extra:
+          example: null
+          type: object
+      type: object
+security:
+  - bearerAuth: []

contrib/example.tengo

@@ -0,0 +1,2 @@
+text := import("text")
+msgText=text.re_replace("matterbridge",msgText,"matterbridge (https://github.com/42wim/matterbridge)")

contrib/outmessage-discordemoji.tengo

@@ -0,0 +1,10 @@
+text := import("text")
+
+// if we're not sending to a discord bridge,
+// then convert custom emoji tags into url's
+if (inProtocol == "discord" && outProtocol != "discord") {
+    rePNG := text.re_compile(`<:.*?:([0-9]+)>`)
+    msgText=rePNG.replace(msgText,"https://cdn.discordapp.com/emojis/$1.png")
+    reGIF := text.re_compile(`<a:.*?:([0-9]+)>`)
+    msgText=reGIF.replace(msgText,"https://cdn.discordapp.com/emojis/$1.gif")
+}

contrib/outmessage-irccolornick.tengo

@@ -0,0 +1,14 @@
+// See https://github.com/42wim/matterbridge/issues/881
+// Generates a colored nick for each msgUsername, with example to filter specific codes 
+
+text := import("text")
+fmt := import("fmt")
+if outProtocol == "irc" {
+    // generate a color for a nick, make sure it isn't 0 or 15
+    colorCode := len(msgUsername)+bytes(msgUsername)[0]%14 + 2
+    // example if we want to use colorCode 3 when we have calculated colorcode 14
+    if colorCode == 14 {
+        colorCode = 3
+    }
+    msgUsername=fmt.sprintf("\x03%02d%s\x0F", colorCode, msgUsername)
+}

contrib/outmessage-irccolors.tengo

@@ -0,0 +1,7 @@
+// See https://github.com/42wim/matterbridge/issues/798
+
+// if we're not sending to an irc bridge we strip the IRC colors
+if outProtocol != "irc" {
+    re := text.re_compile(`\x03(?:\d{1,2}(?:,\d{1,2})?)?|[[:cntrl:]]`)
+    msgText=re.replace(msgText,"")
+}

contrib/remotenickformat-zerowidth.tengo

@@ -0,0 +1,16 @@
+/*
+This script will return the nick except with multi-character usernames
+containing a zero-width space between the first and second character letter.
+
+Single character usernames will be left untouched.
+
+This is useful to prevent remote users from nickalerting
+IRC users of the same name when the remote user speaks.
+
+This result can be used in {TENGO} in RemoteNickFormat.
+*/
+
+result = nick
+if len(nick) > 1 {
+    result = string(nick[0]) + "​" + nick[1:]
+}

contrib/remotenickformat.tengo

@@ -0,0 +1,9 @@
+/*
+This script will return the current time in kitchen format if the protocol (of the remote bridge) isn't irc
+See https://github.com/d5/tengo/blob/master/docs/stdlib-times.md
+This result can be used in {TENGO} in RemoteNickFormat
+*/
+times := import("times")
+if protocol != "irc" {
+   result=times.time_format(times.now(),times.format_kitchen)
+}

docker/arm/Dockerfile

@@ -1,11 +1,9 @@
-FROM cmosh/alpine-arm:edge
+FROM alpine:edge as certs
-ENTRYPOINT ["/bin/matterbridge"]
+RUN apk --update add ca-certificates
 
-COPY . /go/src/github.com/42wim/matterbridge
+FROM scratch
-RUN apk update && apk add go git gcc musl-dev ca-certificates \
+ARG VERSION=1.12.3
-        && cd /go/src/github.com/42wim/matterbridge \
+COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
-        && export GOPATH=/go \
+ADD https://github.com/42wim/matterbridge/releases/download/v${VERSION}/matterbridge-linux-arm /bin/matterbridge
-        && go get \
+RUN chmod +x /bin/matterbridge
-        && go build -x -ldflags "-X main.githash=$(git log --pretty=format:'%h' -n 1)" -o /bin/matterbridge \
+ENTRYPOINT ["/bin/matterbridge"]
-        && rm -rf /go \
-        && apk del --purge git go gcc musl-dev

gateway/bench.tengo

@@ -0,0 +1,5 @@
+text := import("text")
+if text.re_match("blah",msgText) {
+    msgText="replaced by this"
+    msgUsername="fakeuser"
+}

gateway/bridgemap/api.go

@@ -0,0 +1,11 @@
+// +build !noapi
+
+package bridgemap
+
+import (
+	"github.com/42wim/matterbridge/bridge/api"
+)
+
+func init() {
+	FullMap["api"] = api.New
+}

gateway/bridgemap/bdiscord.go

@@ -0,0 +1,12 @@
+// +build !nodiscord
+
+package bridgemap
+
+import (
+	bdiscord "github.com/42wim/matterbridge/bridge/discord"
+)
+
+func init() {
+	FullMap["discord"] = bdiscord.New
+	UserTypingSupport["discord"] = struct{}{}
+}

gateway/bridgemap/bgitter.go

@@ -0,0 +1,11 @@
+// +build !nogitter
+
+package bridgemap
+
+import (
+	bgitter "github.com/42wim/matterbridge/bridge/gitter"
+)
+
+func init() {
+	FullMap["gitter"] = bgitter.New
+}

gateway/bridgemap/birc.go

@@ -0,0 +1,11 @@
+// +build !noirc
+
+package bridgemap
+
+import (
+	birc "github.com/42wim/matterbridge/bridge/irc"
+)
+
+func init() {
+	FullMap["irc"] = birc.New
+}

gateway/bridgemap/bkeybase.go

@@ -0,0 +1,11 @@
+// +build !nokeybase
+
+package bridgemap
+
+import (
+	bkeybase "github.com/42wim/matterbridge/bridge/keybase"
+)
+
+func init() {
+	FullMap["keybase"] = bkeybase.New
+}

gateway/bridgemap/bmatrix.go

@@ -0,0 +1,11 @@
+// +build !nomatrix
+
+package bridgemap
+
+import (
+	bmatrix "github.com/42wim/matterbridge/bridge/matrix"
+)
+
+func init() {
+	FullMap["matrix"] = bmatrix.New
+}

gateway/bridgemap/bmattermost.go

@@ -0,0 +1,11 @@
+// +build !nomattermost
+
+package bridgemap
+
+import (
+	bmattermost "github.com/42wim/matterbridge/bridge/mattermost"
+)
+
+func init() {
+	FullMap["mattermost"] = bmattermost.New
+}

gateway/bridgemap/bmsteams.go

@@ -0,0 +1,11 @@
+// +build !nomsteams
+
+package bridgemap
+
+import (
+	bmsteams "github.com/42wim/matterbridge/bridge/msteams"
+)
+
+func init() {
+	FullMap["msteams"] = bmsteams.New
+}

gateway/bridgemap/bnctalk.go

@@ -0,0 +1,11 @@
+// +build !nonctalk
+
+package bridgemap
+
+import (
+	btalk "github.com/42wim/matterbridge/bridge/nctalk"
+)
+
+func init() {
+	FullMap["nctalk"] = btalk.New
+}

gateway/bridgemap/bridgemap.go

@@ -0,0 +1,10 @@
+package bridgemap
+
+import (
+	"github.com/42wim/matterbridge/bridge"
+)
+
+var (
+	FullMap           = map[string]bridge.Factory{}
+	UserTypingSupport = map[string]struct{}{}
+)

gateway/bridgemap/brocketchat.go

@@ -0,0 +1,11 @@
+// +build !norocketchat
+
+package bridgemap
+
+import (
+	brocketchat "github.com/42wim/matterbridge/bridge/rocketchat"
+)
+
+func init() {
+	FullMap["rocketchat"] = brocketchat.New
+}

gateway/bridgemap/bslack.go

@@ -0,0 +1,13 @@
+// +build !noslack
+
+package bridgemap
+
+import (
+	bslack "github.com/42wim/matterbridge/bridge/slack"
+)
+
+func init() {
+	FullMap["slack-legacy"] = bslack.NewLegacy
+	FullMap["slack"] = bslack.New
+	UserTypingSupport["slack"] = struct{}{}
+}

gateway/bridgemap/bsshchat.go

@@ -0,0 +1,11 @@
+// +build !nosshchat
+
+package bridgemap
+
+import (
+	bsshchat "github.com/42wim/matterbridge/bridge/sshchat"
+)
+
+func init() {
+	FullMap["sshchat"] = bsshchat.New
+}

gateway/bridgemap/bsteam.go

@@ -0,0 +1,11 @@
+// +build !nosteam
+
+package bridgemap
+
+import (
+	bsteam "github.com/42wim/matterbridge/bridge/steam"
+)
+
+func init() {
+	FullMap["steam"] = bsteam.New
+}

gateway/bridgemap/btelegram.go

@@ -0,0 +1,11 @@
+// +build !notelegram
+
+package bridgemap
+
+import (
+	btelegram "github.com/42wim/matterbridge/bridge/telegram"
+)
+
+func init() {
+	FullMap["telegram"] = btelegram.New
+}

gateway/bridgemap/bwhatsapp.go

@@ -0,0 +1,11 @@
+// +build !nowhatsapp
+
+package bridgemap
+
+import (
+	bwhatsapp "github.com/42wim/matterbridge/bridge/whatsapp"
+)
+
+func init() {
+	FullMap["whatsapp"] = bwhatsapp.New
+}

gateway/bridgemap/bxmpp.go

@@ -0,0 +1,11 @@
+// +build !noxmpp
+
+package bridgemap
+
+import (
+	bxmpp "github.com/42wim/matterbridge/bridge/xmpp"
+)
+
+func init() {
+	FullMap["xmpp"] = bxmpp.New
+}

gateway/bridgemap/bzulip.go

@@ -0,0 +1,11 @@
+// +build !nozulip
+
+package bridgemap
+
+import (
+	bzulip "github.com/42wim/matterbridge/bridge/zulip"
+)
+
+func init() {
+	FullMap["zulip"] = bzulip.New
+}

gateway/gateway.go

@@ -1,41 +1,25 @@
 package gateway
 
 import (
-	"bytes"
-	"fmt"
 	"io/ioutil"
-	"net/http"
 	"os"
-
-	"github.com/42wim/matterbridge/bridge"
-	"github.com/42wim/matterbridge/bridge/api"
-	"github.com/42wim/matterbridge/bridge/config"
-	bdiscord "github.com/42wim/matterbridge/bridge/discord"
-	bgitter "github.com/42wim/matterbridge/bridge/gitter"
-	birc "github.com/42wim/matterbridge/bridge/irc"
-	bmatrix "github.com/42wim/matterbridge/bridge/matrix"
-	bmattermost "github.com/42wim/matterbridge/bridge/mattermost"
-	brocketchat "github.com/42wim/matterbridge/bridge/rocketchat"
-	bslack "github.com/42wim/matterbridge/bridge/slack"
-	bsshchat "github.com/42wim/matterbridge/bridge/sshchat"
-	bsteam "github.com/42wim/matterbridge/bridge/steam"
-	btelegram "github.com/42wim/matterbridge/bridge/telegram"
-	bxmpp "github.com/42wim/matterbridge/bridge/xmpp"
-	bzulip "github.com/42wim/matterbridge/bridge/zulip"
-	"github.com/hashicorp/golang-lru"
-	log "github.com/sirupsen/logrus"
-	//	"github.com/davecgh/go-spew/spew"
-	"crypto/sha1"
-	"path/filepath"
 	"regexp"
 	"strings"
 	"time"
 
-	"github.com/peterhellberg/emojilib"
+	"github.com/42wim/matterbridge/bridge"
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/42wim/matterbridge/internal"
+	"github.com/d5/tengo/v2"
+	"github.com/d5/tengo/v2/stdlib"
+	lru "github.com/hashicorp/golang-lru"
+	"github.com/matterbridge/emoji"
+	"github.com/sirupsen/logrus"
 )
 
 type Gateway struct {
-	*config.Config
+	config.Config
+
 	Router         *Router
 	MyConfig       *config.Gateway
 	Bridges        map[string]*bridge.Bridge
@@ -44,6 +28,8 @@ type Gateway struct {
 	Message        chan config.Message
 	Name           string
 	Messages       *lru.Cache
+
+	logger *logrus.Entry
 }
 
 type BrMsgID struct {
@@ -52,59 +38,95 @@ type BrMsgID struct {
 	ChannelID string
 }
 
-var flog *log.Entry
+const apiProtocol = "api"
 
-var bridgeMap = map[string]bridge.Factory{
+// New creates a new Gateway object associated with the specified router and
-	"api":        api.New,
+// following the given configuration.
-	"discord":    bdiscord.New,
+func New(rootLogger *logrus.Logger, cfg *config.Gateway, r *Router) *Gateway {
-	"gitter":     bgitter.New,
+	logger := rootLogger.WithFields(logrus.Fields{"prefix": "gateway"})
-	"irc":        birc.New,
-	"mattermost": bmattermost.New,
-	"matrix":     bmatrix.New,
-	"rocketchat": brocketchat.New,
-	"slack":      bslack.New,
-	"sshchat":    bsshchat.New,
-	"steam":      bsteam.New,
-	"telegram":   btelegram.New,
-	"xmpp":       bxmpp.New,
-	"zulip":      bzulip.New,
-}
 
-func init() {
-	flog = log.WithFields(log.Fields{"prefix": "gateway"})
-}
-
-func New(cfg config.Gateway, r *Router) *Gateway {
-	gw := &Gateway{Channels: make(map[string]*config.ChannelInfo), Message: r.Message,
-		Router: r, Bridges: make(map[string]*bridge.Bridge), Config: r.Config}
 	cache, _ := lru.New(5000)
-	gw.Messages = cache
+	gw := &Gateway{
-	gw.AddConfig(&cfg)
+		Channels: make(map[string]*config.ChannelInfo),
+		Message:  r.Message,
+		Router:   r,
+		Bridges:  make(map[string]*bridge.Bridge),
+		Config:   r.Config,
+		Messages: cache,
+		logger:   logger,
+	}
+	if err := gw.AddConfig(cfg); err != nil {
+		logger.Errorf("Failed to add configuration to gateway: %#v", err)
+	}
 	return gw
 }
 
+// FindCanonicalMsgID returns the ID under which a message was stored in the cache.
+func (gw *Gateway) FindCanonicalMsgID(protocol string, mID string) string {
+	ID := protocol + " " + mID
+	if gw.Messages.Contains(ID) {
+		return mID
+	}
+
+	// If not keyed, iterate through cache for downstream, and infer upstream.
+	for _, mid := range gw.Messages.Keys() {
+		v, _ := gw.Messages.Peek(mid)
+		ids := v.([]*BrMsgID)
+		for _, downstreamMsgObj := range ids {
+			if ID == downstreamMsgObj.ID {
+				return strings.Replace(mid.(string), protocol+" ", "", 1)
+			}
+		}
+	}
+	return ""
+}
+
+// AddBridge sets up a new bridge in the gateway object with the specified configuration.
 func (gw *Gateway) AddBridge(cfg *config.Bridge) error {
 	br := gw.Router.getBridge(cfg.Account)
 	if br == nil {
+		gw.checkConfig(cfg)
 		br = bridge.New(cfg)
 		br.Config = gw.Router.Config
-		br.General = &gw.General
+		br.General = &gw.BridgeValues().General
-		// set logging
+		br.Log = gw.logger.WithFields(logrus.Fields{"prefix": br.Protocol})
-		br.Log = log.WithFields(log.Fields{"prefix": "bridge"})
+		brconfig := &bridge.Config{
-		brconfig := &bridge.Config{Remote: gw.Message, Log: log.WithFields(log.Fields{"prefix": br.Protocol}), Bridge: br}
+			Remote: gw.Message,
+			Bridge: br,
+		}
 		// add the actual bridger for this protocol to this bridge using the bridgeMap
-		br.Bridger = bridgeMap[br.Protocol](brconfig)
+		if _, ok := gw.Router.BridgeMap[br.Protocol]; !ok {
+			gw.logger.Fatalf("Incorrect protocol %s specified in gateway configuration %s, exiting.", br.Protocol, cfg.Account)
+		}
+		br.Bridger = gw.Router.BridgeMap[br.Protocol](brconfig)
 	}
 	gw.mapChannelsToBridge(br)
 	gw.Bridges[cfg.Account] = br
 	return nil
 }
 
+func (gw *Gateway) checkConfig(cfg *config.Bridge) {
+	match := false
+	for _, key := range gw.Router.Config.Viper().AllKeys() {
+		if strings.HasPrefix(key, strings.ToLower(cfg.Account)) {
+			match = true
+			break
+		}
+	}
+	if !match {
+		gw.logger.Fatalf("Account %s defined in gateway %s but no configuration found, exiting.", cfg.Account, gw.Name)
+	}
+}
+
+// AddConfig associates a new configuration with the gateway object.
 func (gw *Gateway) AddConfig(cfg *config.Gateway) error {
 	gw.Name = cfg.Name
 	gw.MyConfig = cfg
-	gw.mapChannels()
+	if err := gw.mapChannels(); err != nil {
+		gw.logger.Errorf("mapChannels() failed: %s", err)
+	}
 	for _, br := range append(gw.MyConfig.In, append(gw.MyConfig.InOut, gw.MyConfig.Out...)...) {
+		br := br //scopelint
 		err := gw.AddBridge(&br)
 		if err != nil {
 			return err
@@ -122,33 +144,51 @@ func (gw *Gateway) mapChannelsToBridge(br *bridge.Bridge) {
 }
 
 func (gw *Gateway) reconnectBridge(br *bridge.Bridge) {
-	br.Disconnect()
+	if err := br.Disconnect(); err != nil {
+		gw.logger.Errorf("Disconnect() %s failed: %s", br.Account, err)
+	}
 	time.Sleep(time.Second * 5)
 RECONNECT:
-	flog.Infof("Reconnecting %s", br.Account)
+	gw.logger.Infof("Reconnecting %s", br.Account)
 	err := br.Connect()
 	if err != nil {
-		flog.Errorf("Reconnection failed: %s. Trying again in 60 seconds", err)
+		gw.logger.Errorf("Reconnection failed: %s. Trying again in 60 seconds", err)
 		time.Sleep(time.Second * 60)
 		goto RECONNECT
 	}
 	br.Joined = make(map[string]bool)
-	br.JoinChannels()
+	if err := br.JoinChannels(); err != nil {
+		gw.logger.Errorf("JoinChannels() %s failed: %s", br.Account, err)
+	}
 }
 
 func (gw *Gateway) mapChannelConfig(cfg []config.Bridge, direction string) {
 	for _, br := range cfg {
-		if isApi(br.Account) {
+		if isAPI(br.Account) {
-			br.Channel = "api"
+			br.Channel = apiProtocol
 		}
 		// make sure to lowercase irc channels in config #348
 		if strings.HasPrefix(br.Account, "irc.") {
 			br.Channel = strings.ToLower(br.Channel)
 		}
+		if strings.HasPrefix(br.Account, "mattermost.") && strings.HasPrefix(br.Channel, "#") {
+			gw.logger.Errorf("Mattermost channels do not start with a #: remove the # in %s", br.Channel)
+			os.Exit(1)
+		}
+		if strings.HasPrefix(br.Account, "zulip.") && !strings.Contains(br.Channel, "/topic:") {
+			gw.logger.Errorf("Breaking change, since matterbridge 1.14.0 zulip channels need to specify the topic with channel/topic:mytopic in %s of %s", br.Channel, br.Account)
+			os.Exit(1)
+		}
 		ID := br.Channel + br.Account
 		if _, ok := gw.Channels[ID]; !ok {
-			channel := &config.ChannelInfo{Name: br.Channel, Direction: direction, ID: ID, Options: br.Options, Account: br.Account,
+			channel := &config.ChannelInfo{
-				SameChannel: make(map[string]bool)}
+				Name:        br.Channel,
+				Direction:   direction,
+				ID:          ID,
+				Options:     br.Options,
+				Account:     br.Account,
+				SameChannel: make(map[string]bool),
+			}
 			channel.SameChannel[gw.Name] = br.SameChannel
 			gw.Channels[channel.ID] = channel
 		} else {
@@ -172,14 +212,25 @@ func (gw *Gateway) getDestChannel(msg *config.Message, dest bridge.Bridge) []con
 	var channels []config.ChannelInfo
 
 	// for messages received from the api check that the gateway is the specified one
-	if msg.Protocol == "api" && gw.Name != msg.Gateway {
+	if msg.Protocol == apiProtocol && gw.Name != msg.Gateway {
+		return channels
+	}
+
+	// discord join/leave is for the whole bridge, isn't a per channel join/leave
+	if msg.Event == config.EventJoinLeave && getProtocol(msg) == "discord" && msg.Channel == "" {
+		for _, channel := range gw.Channels {
+			if channel.Account == dest.Account && strings.Contains(channel.Direction, "out") &&
+				gw.validGatewayDest(msg) {
+				channels = append(channels, *channel)
+			}
+		}
 		return channels
 	}
 
 	// if source channel is in only, do nothing
 	for _, channel := range gw.Channels {
 		// lookup the channel from the message
-		if channel.ID == getChannelID(*msg) {
+		if channel.ID == getChannelID(msg) {
 			// we only have destinations if the original message is from an "in" (sending) channel
 			if !strings.Contains(channel.Direction, "in") {
 				return channels
@@ -188,105 +239,55 @@ func (gw *Gateway) getDestChannel(msg *config.Message, dest bridge.Bridge) []con
 		}
 	}
 	for _, channel := range gw.Channels {
-		if _, ok := gw.Channels[getChannelID(*msg)]; !ok {
+		if _, ok := gw.Channels[getChannelID(msg)]; !ok {
 			continue
 		}
 
-		// do samechannelgateway flogic
+		// do samechannelgateway logic
 		if channel.SameChannel[msg.Gateway] {
 			if msg.Channel == channel.Name && msg.Account != dest.Account {
 				channels = append(channels, *channel)
 			}
 			continue
 		}
-		if strings.Contains(channel.Direction, "out") && channel.Account == dest.Account && gw.validGatewayDest(msg, channel) {
+		if strings.Contains(channel.Direction, "out") && channel.Account == dest.Account && gw.validGatewayDest(msg) {
 			channels = append(channels, *channel)
 		}
 	}
 	return channels
 }
 
-func (gw *Gateway) handleMessage(msg config.Message, dest *bridge.Bridge) []*BrMsgID {
+func (gw *Gateway) getDestMsgID(msgID string, dest *bridge.Bridge, channel *config.ChannelInfo) string {
-	var brMsgIDs []*BrMsgID
+	if res, ok := gw.Messages.Get(msgID); ok {
-
+		IDs := res.([]*BrMsgID)
-	// if we have an attached file, or other info
+		for _, id := range IDs {
-	if msg.Extra != nil {
+			// check protocol, bridge name and channelname
-		if len(msg.Extra[config.EVENT_FILE_FAILURE_SIZE]) != 0 {
+			// for people that reuse the same bridge multiple times. see #342
-			if msg.Text == "" {
+			if dest.Protocol == id.br.Protocol && dest.Name == id.br.Name && channel.ID == id.ChannelID {
-				return brMsgIDs
+				return strings.Replace(id.ID, dest.Protocol+" ", "", 1)
 			}
 		}
 	}
+	return ""
+}
 
-	// Avatar downloads are only relevant for telegram and mattermost for now
+// ignoreTextEmpty returns true if we need to ignore a message with an empty text.
-	if msg.Event == config.EVENT_AVATAR_DOWNLOAD {
+func (gw *Gateway) ignoreTextEmpty(msg *config.Message) bool {
-		if dest.Protocol != "mattermost" &&
+	if msg.Text != "" {
-			dest.Protocol != "telegram" {
+		return false
-			return brMsgIDs
-		}
 	}
-
+	if msg.Event == config.EventUserTyping {
-	// only relay join/part when configured
+		return false
-	if msg.Event == config.EVENT_JOIN_LEAVE && !gw.Bridges[dest.Account].GetBool("ShowJoinPart") {
-		return brMsgIDs
 	}
-
+	// we have an attachment or actual bytes, do not ignore
-	// only relay topic change when configured
+	if msg.Extra != nil &&
-	if msg.Event == config.EVENT_TOPIC_CHANGE && !gw.Bridges[dest.Account].GetBool("ShowTopicChange") {
+		(msg.Extra["attachments"] != nil ||
-		return brMsgIDs
+			len(msg.Extra["file"]) > 0 ||
+			len(msg.Extra[config.EventFileFailureSize]) > 0) {
+		return false
 	}
-
+	gw.logger.Debugf("ignoring empty message %#v from %s", msg, msg.Account)
-	// broadcast to every out channel (irc QUIT)
+	return true
-	if msg.Channel == "" && msg.Event != config.EVENT_JOIN_LEAVE {
-		flog.Debug("empty channel")
-		return brMsgIDs
-	}
-
-	originchannel := msg.Channel
-	origmsg := msg
-	channels := gw.getDestChannel(&msg, *dest)
-	for _, channel := range channels {
-		// Only send the avatar download event to ourselves.
-		if msg.Event == config.EVENT_AVATAR_DOWNLOAD {
-			if channel.ID != getChannelID(origmsg) {
-				continue
-			}
-		} else {
-			// do not send to ourself for any other event
-			if channel.ID == getChannelID(origmsg) {
-				continue
-			}
-		}
-		flog.Debugf("=> Sending %#v from %s (%s) to %s (%s)", msg, msg.Account, originchannel, dest.Account, channel.Name)
-		msg.Channel = channel.Name
-		msg.Avatar = gw.modifyAvatar(origmsg, dest)
-		msg.Username = gw.modifyUsername(origmsg, dest)
-		msg.ID = ""
-		if res, ok := gw.Messages.Get(origmsg.ID); ok {
-			IDs := res.([]*BrMsgID)
-			for _, id := range IDs {
-				// check protocol, bridge name and channelname
-				// for people that reuse the same bridge multiple times. see #342
-				if dest.Protocol == id.br.Protocol && dest.Name == id.br.Name && channel.ID == id.ChannelID {
-					msg.ID = id.ID
-				}
-			}
-		}
-		// for api we need originchannel as channel
-		if dest.Protocol == "api" {
-			msg.Channel = originchannel
-		}
-		mID, err := dest.Send(msg)
-		if err != nil {
-			flog.Error(err)
-		}
-		// append the message ID (mID) from this bridge (dest) to our brMsgIDs slice
-		if mID != "" {
-			flog.Debugf("mID %s: %s", dest.Account, mID)
-			brMsgIDs = append(brMsgIDs, &BrMsgID{dest, mID, channel.ID})
-		}
-	}
-	return brMsgIDs
 }
 
 func (gw *Gateway) ignoreMessage(msg *config.Message) bool {
@@ -295,65 +296,31 @@ func (gw *Gateway) ignoreMessage(msg *config.Message) bool {
 		return true
 	}
 
-	// check if we need to ignore a empty message
+	igNicks := strings.Fields(gw.Bridges[msg.Account].GetString("IgnoreNicks"))
-	if msg.Text == "" {
+	igMessages := strings.Fields(gw.Bridges[msg.Account].GetString("IgnoreMessages"))
-		// we have an attachment or actual bytes, do not ignore
+	if gw.ignoreTextEmpty(msg) || gw.ignoreText(msg.Username, igNicks) || gw.ignoreText(msg.Text, igMessages) {
-		if msg.Extra != nil &&
-			(msg.Extra["attachments"] != nil ||
-				len(msg.Extra["file"]) > 0 ||
-				len(msg.Extra[config.EVENT_FILE_FAILURE_SIZE]) > 0) {
-			return false
-		}
-		flog.Debugf("ignoring empty message %#v from %s", msg, msg.Account)
 		return true
 	}
 
-	// is the username in IgnoreNicks field
-	for _, entry := range strings.Fields(gw.Bridges[msg.Account].GetString("IgnoreNicks")) {
-		if msg.Username == entry {
-			flog.Debugf("ignoring %s from %s", msg.Username, msg.Account)
-			return true
-		}
-	}
-
-	// does the message match regex in IgnoreMessages field
-	// TODO do not compile regexps everytime
-	for _, entry := range strings.Fields(gw.Bridges[msg.Account].GetString("IgnoreMessages")) {
-		if entry != "" {
-			re, err := regexp.Compile(entry)
-			if err != nil {
-				flog.Errorf("incorrect regexp %s for %s", entry, msg.Account)
-				continue
-			}
-			if re.MatchString(msg.Text) {
-				flog.Debugf("matching %s. ignoring %s from %s", entry, msg.Text, msg.Account)
-				return true
-			}
-		}
-	}
 	return false
 }
 
-func (gw *Gateway) modifyUsername(msg config.Message, dest *bridge.Bridge) string {
+func (gw *Gateway) modifyUsername(msg *config.Message, dest *bridge.Bridge) string {
-	br := gw.Bridges[msg.Account]
+	if dest.GetBool("StripNick") {
-	msg.Protocol = br.Protocol
-	if gw.Config.General.StripNick || dest.GetBool("StripNick") {
 		re := regexp.MustCompile("[^a-zA-Z0-9]+")
 		msg.Username = re.ReplaceAllString(msg.Username, "")
 	}
 	nick := dest.GetString("RemoteNickFormat")
-	if nick == "" {
-		nick = gw.Config.General.RemoteNickFormat
-	}
 
 	// loop to replace nicks
+	br := gw.Bridges[msg.Account]
 	for _, outer := range br.GetStringSlice2D("ReplaceNicks") {
 		search := outer[0]
 		replace := outer[1]
 		// TODO move compile to bridge init somewhere
 		re, err := regexp.Compile(search)
 		if err != nil {
-			flog.Errorf("regexp in %s failed: %s", msg.Account, err)
+			gw.logger.Errorf("regexp in %s failed: %s", msg.Account, err)
 			break
 		}
 		msg.Username = re.ReplaceAllString(msg.Username, replace)
@@ -374,16 +341,20 @@ func (gw *Gateway) modifyUsername(msg config.Message, dest *bridge.Bridge) strin
 
 	nick = strings.Replace(nick, "{BRIDGE}", br.Name, -1)
 	nick = strings.Replace(nick, "{PROTOCOL}", br.Protocol, -1)
+	nick = strings.Replace(nick, "{GATEWAY}", gw.Name, -1)
 	nick = strings.Replace(nick, "{LABEL}", br.GetString("Label"), -1)
 	nick = strings.Replace(nick, "{NICK}", msg.Username, -1)
+	nick = strings.Replace(nick, "{CHANNEL}", msg.Channel, -1)
+	tengoNick, err := gw.modifyUsernameTengo(msg, br)
+	if err != nil {
+		gw.logger.Errorf("modifyUsernameTengo error: %s", err)
+	}
+	nick = strings.Replace(nick, "{TENGO}", tengoNick, -1) //nolint:gocritic
 	return nick
 }
 
-func (gw *Gateway) modifyAvatar(msg config.Message, dest *bridge.Bridge) string {
+func (gw *Gateway) modifyAvatar(msg *config.Message, dest *bridge.Bridge) string {
-	iconurl := gw.Config.General.IconURL
+	iconurl := dest.GetString("IconURL")
-	if iconurl == "" {
-		iconurl = dest.GetString("IconURL")
-	}
 	iconurl = strings.Replace(iconurl, "{NICK}", msg.Username, -1)
 	if msg.Avatar == "" {
 		msg.Avatar = iconurl
@@ -392,8 +363,15 @@ func (gw *Gateway) modifyAvatar(msg config.Message, dest *bridge.Bridge) string
 }
 
 func (gw *Gateway) modifyMessage(msg *config.Message) {
+	if err := modifyMessageTengo(gw.BridgeValues().General.TengoModifyMessage, msg); err != nil {
+		gw.logger.Errorf("TengoModifyMessage failed: %s", err)
+	}
+	if err := modifyMessageTengo(gw.BridgeValues().Tengo.Message, msg); err != nil {
+		gw.logger.Errorf("Tengo.Message failed: %s", err)
+	}
+
 	// replace :emoji: to unicode
-	msg.Text = emojilib.Replace(msg.Text)
+	msg.Text = emoji.Sprint(msg.Text)
 
 	br := gw.Bridges[msg.Account]
 	// loop to replace messages
@@ -403,106 +381,226 @@ func (gw *Gateway) modifyMessage(msg *config.Message) {
 		// TODO move compile to bridge init somewhere
 		re, err := regexp.Compile(search)
 		if err != nil {
-			flog.Errorf("regexp in %s failed: %s", msg.Account, err)
+			gw.logger.Errorf("regexp in %s failed: %s", msg.Account, err)
 			break
 		}
 		msg.Text = re.ReplaceAllString(msg.Text, replace)
 	}
 
+	gw.handleExtractNicks(msg)
+
 	// messages from api have Gateway specified, don't overwrite
-	if msg.Protocol != "api" {
+	if msg.Protocol != apiProtocol {
 		msg.Gateway = gw.Name
 	}
 }
 
-// handleFiles uploads or places all files on the given msg to the MediaServer and
+// SendMessage sends a message (with specified parentID) to the channel on the selected
-// adds the new URL of the file on the MediaServer onto the given msg.
+// destination bridge and returns a message ID or an error.
-func (gw *Gateway) handleFiles(msg *config.Message) {
+func (gw *Gateway) SendMessage(
-	reg := regexp.MustCompile("[^a-zA-Z0-9]+")
+	rmsg *config.Message,
-
+	dest *bridge.Bridge,
-	// If we don't have a attachfield or we don't have a mediaserver configured return
+	channel *config.ChannelInfo,
-	if msg.Extra == nil || (gw.Config.General.MediaServerUpload == "" && gw.Config.General.MediaDownloadPath == "") {
+	canonicalParentMsgID string,
-		return
+) (string, error) {
-	}
+	msg := *rmsg
-
+	// Only send the avatar download event to ourselves.
-	// If we don't have files, nothing to upload.
+	if msg.Event == config.EventAvatarDownload {
-	if len(msg.Extra["file"]) == 0 {
+		if channel.ID != getChannelID(rmsg) {
-		return
+			return "", nil
-	}
+		}
-
+	} else {
-	client := &http.Client{
+		// do not send to ourself for any other event
-		Timeout: time.Second * 5,
+		if channel.ID == getChannelID(rmsg) {
-	}
+			return "", nil
-
-	for i, f := range msg.Extra["file"] {
-		fi := f.(config.FileInfo)
-		ext := filepath.Ext(fi.Name)
-		fi.Name = fi.Name[0 : len(fi.Name)-len(ext)]
-		fi.Name = reg.ReplaceAllString(fi.Name, "_")
-		fi.Name = fi.Name + ext
-
-		sha1sum := fmt.Sprintf("%x", sha1.Sum(*fi.Data))[:8]
-
-		if gw.Config.General.MediaServerUpload != "" {
-			// Use MediaServerUpload. Upload using a PUT HTTP request and basicauth.
-
-			url := gw.Config.General.MediaServerUpload + "/" + sha1sum + "/" + fi.Name
-
-			req, err := http.NewRequest("PUT", url, bytes.NewReader(*fi.Data))
-			if err != nil {
-				flog.Errorf("mediaserver upload failed, could not create request: %#v", err)
-				continue
-			}
-
-			flog.Debugf("mediaserver upload url: %s", url)
-
-			req.Header.Set("Content-Type", "binary/octet-stream")
-			_, err = client.Do(req)
-			if err != nil {
-				flog.Errorf("mediaserver upload failed, could not Do request: %#v", err)
-				continue
-			}
-		} else {
-			// Use MediaServerPath. Place the file on the current filesystem.
-
-			dir := gw.Config.General.MediaDownloadPath + "/" + sha1sum
-			err := os.Mkdir(dir, os.ModePerm)
-			if err != nil && !os.IsExist(err) {
-				flog.Errorf("mediaserver path failed, could not mkdir: %s %#v", err, err)
-				continue
-			}
-
-			path := dir + "/" + fi.Name
-			flog.Debugf("mediaserver path placing file: %s", path)
-
-			err = ioutil.WriteFile(path, *fi.Data, os.ModePerm)
-			if err != nil {
-				flog.Errorf("mediaserver path failed, could not writefile: %s %#v", err, err)
-				continue
-			}
 		}
-
-		// Download URL.
-		durl := gw.Config.General.MediaServerDownload + "/" + sha1sum + "/" + fi.Name
-
-		flog.Debugf("mediaserver download URL = %s", durl)
-
-		// We uploaded/placed the file successfully. Add the SHA and URL.
-		extra := msg.Extra["file"][i].(config.FileInfo)
-		extra.URL = durl
-		extra.SHA = sha1sum
-		msg.Extra["file"][i] = extra
 	}
+
+	// Too noisy to log like other events
+	if msg.Event != config.EventUserTyping {
+		gw.logger.Debugf("=> Sending %#v from %s (%s) to %s (%s)", msg, msg.Account, rmsg.Channel, dest.Account, channel.Name)
+	}
+
+	msg.Channel = channel.Name
+	msg.Avatar = gw.modifyAvatar(rmsg, dest)
+	msg.Username = gw.modifyUsername(rmsg, dest)
+
+	msg.ID = gw.getDestMsgID(rmsg.Protocol+" "+rmsg.ID, dest, channel)
+
+	// for api we need originchannel as channel
+	if dest.Protocol == apiProtocol {
+		msg.Channel = rmsg.Channel
+	}
+
+	msg.ParentID = gw.getDestMsgID(rmsg.Protocol+" "+canonicalParentMsgID, dest, channel)
+	if msg.ParentID == "" {
+		msg.ParentID = canonicalParentMsgID
+	}
+
+	// if the parentID is still empty and we have a parentID set in the original message
+	// this means that we didn't find it in the cache so set it "msg-parent-not-found"
+	if msg.ParentID == "" && rmsg.ParentID != "" {
+		msg.ParentID = "msg-parent-not-found"
+	}
+
+	err := gw.modifySendMessageTengo(rmsg, &msg, dest)
+	if err != nil {
+		gw.logger.Errorf("modifySendMessageTengo: %s", err)
+	}
+
+	// if we are using mattermost plugin account, send messages to MattermostPlugin channel
+	// that can be picked up by the mattermost matterbridge plugin
+	if dest.Account == "mattermost.plugin" {
+		gw.Router.MattermostPlugin <- msg
+	}
+
+	mID, err := dest.Send(msg)
+	if err != nil {
+		return mID, err
+	}
+
+	// append the message ID (mID) from this bridge (dest) to our brMsgIDs slice
+	if mID != "" {
+		gw.logger.Debugf("mID %s: %s", dest.Account, mID)
+		return mID, nil
+		//brMsgIDs = append(brMsgIDs, &BrMsgID{dest, dest.Protocol + " " + mID, channel.ID})
+	}
+	return "", nil
 }
 
-func (gw *Gateway) validGatewayDest(msg *config.Message, channel *config.ChannelInfo) bool {
+func (gw *Gateway) validGatewayDest(msg *config.Message) bool {
 	return msg.Gateway == gw.Name
 }
 
-func getChannelID(msg config.Message) string {
+func getChannelID(msg *config.Message) string {
 	return msg.Channel + msg.Account
 }
 
-func isApi(account string) bool {
+func isAPI(account string) bool {
 	return strings.HasPrefix(account, "api.")
 }
+
+// ignoreText returns true if text matches any of the input regexes.
+func (gw *Gateway) ignoreText(text string, input []string) bool {
+	for _, entry := range input {
+		if entry == "" {
+			continue
+		}
+		// TODO do not compile regexps everytime
+		re, err := regexp.Compile(entry)
+		if err != nil {
+			gw.logger.Errorf("incorrect regexp %s", entry)
+			continue
+		}
+		if re.MatchString(text) {
+			gw.logger.Debugf("matching %s. ignoring %s", entry, text)
+			return true
+		}
+	}
+	return false
+}
+
+func getProtocol(msg *config.Message) string {
+	p := strings.Split(msg.Account, ".")
+	return p[0]
+}
+
+func modifyMessageTengo(filename string, msg *config.Message) error {
+	if filename == "" {
+		return nil
+	}
+	res, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return err
+	}
+	s := tengo.NewScript(res)
+	s.SetImports(stdlib.GetModuleMap(stdlib.AllModuleNames()...))
+	_ = s.Add("msgText", msg.Text)
+	_ = s.Add("msgUsername", msg.Username)
+	_ = s.Add("msgAccount", msg.Account)
+	_ = s.Add("msgChannel", msg.Channel)
+	c, err := s.Compile()
+	if err != nil {
+		return err
+	}
+	if err := c.Run(); err != nil {
+		return err
+	}
+	msg.Text = c.Get("msgText").String()
+	msg.Username = c.Get("msgUsername").String()
+	return nil
+}
+
+func (gw *Gateway) modifyUsernameTengo(msg *config.Message, br *bridge.Bridge) (string, error) {
+	filename := gw.BridgeValues().Tengo.RemoteNickFormat
+	if filename == "" {
+		return "", nil
+	}
+	res, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return "", err
+	}
+	s := tengo.NewScript(res)
+	s.SetImports(stdlib.GetModuleMap(stdlib.AllModuleNames()...))
+	_ = s.Add("result", "")
+	_ = s.Add("msgText", msg.Text)
+	_ = s.Add("msgUsername", msg.Username)
+	_ = s.Add("nick", msg.Username)
+	_ = s.Add("msgAccount", msg.Account)
+	_ = s.Add("msgChannel", msg.Channel)
+	_ = s.Add("channel", msg.Channel)
+	_ = s.Add("msgProtocol", msg.Protocol)
+	_ = s.Add("remoteAccount", br.Account)
+	_ = s.Add("protocol", br.Protocol)
+	_ = s.Add("bridge", br.Name)
+	_ = s.Add("gateway", gw.Name)
+	c, err := s.Compile()
+	if err != nil {
+		return "", err
+	}
+	if err := c.Run(); err != nil {
+		return "", err
+	}
+	return c.Get("result").String(), nil
+}
+
+func (gw *Gateway) modifySendMessageTengo(origmsg *config.Message, msg *config.Message, br *bridge.Bridge) error {
+	filename := gw.BridgeValues().Tengo.OutMessage
+	var res []byte
+	var err error
+	if filename == "" {
+		res, err = internal.Asset("tengo/outmessage.tengo")
+		if err != nil {
+			return err
+		}
+	} else {
+		res, err = ioutil.ReadFile(filename)
+		if err != nil {
+			return err
+		}
+	}
+	s := tengo.NewScript(res)
+	s.SetImports(stdlib.GetModuleMap(stdlib.AllModuleNames()...))
+	_ = s.Add("inAccount", origmsg.Account)
+	_ = s.Add("inProtocol", origmsg.Protocol)
+	_ = s.Add("inChannel", origmsg.Channel)
+	_ = s.Add("inGateway", origmsg.Gateway)
+	_ = s.Add("inEvent", origmsg.Event)
+	_ = s.Add("outAccount", br.Account)
+	_ = s.Add("outProtocol", br.Protocol)
+	_ = s.Add("outChannel", msg.Channel)
+	_ = s.Add("outGateway", gw.Name)
+	_ = s.Add("outEvent", msg.Event)
+	_ = s.Add("msgText", msg.Text)
+	_ = s.Add("msgUsername", msg.Username)
+	c, err := s.Compile()
+	if err != nil {
+		return err
+	}
+	if err := c.Run(); err != nil {
+		return err
+	}
+	msg.Text = c.Get("msgText").String()
+	msg.Username = c.Get("msgUsername").String()
+	return nil
+}

gateway/gateway_test.go

@@ -2,20 +2,28 @@ package gateway
 
 import (
 	"fmt"
+	"io/ioutil"
 	"strconv"
+	"testing"
 
 	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/42wim/matterbridge/gateway/bridgemap"
+	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
-
+	"github.com/stretchr/testify/suite"
-	"testing"
 )
 
 var testconfig = []byte(`
 [irc.freenode]
+server=""
 [mattermost.test]
+server=""
 [gitter.42wim]
+server=""
 [discord.test]
+server=""
 [slack.test]
+server=""
 
 [[gateway]]
     name = "bridge1"
@@ -41,10 +49,15 @@ var testconfig = []byte(`
 
 var testconfig2 = []byte(`
 [irc.freenode]
+server=""
 [mattermost.test]
+server=""
 [gitter.42wim]
+server=""
 [discord.test]
+server=""
 [slack.test]
+server=""
 
 [[gateway]]
     name = "bridge1"
@@ -84,8 +97,11 @@ var testconfig2 = []byte(`
 
 var testconfig3 = []byte(`
 [irc.zzz]
+server=""
 [telegram.zzz]
+server=""
 [slack.zzz]
+server=""
 [[gateway]]
 name="bridge"
 enable=true
@@ -152,9 +168,17 @@ enable=true
     channel="--333333333333"
 `)
 
+const (
+	ircTestAccount   = "irc.zzz"
+	tgTestAccount    = "telegram.zzz"
+	slackTestAccount = "slack.zzz"
+)
+
 func maketestRouter(input []byte) *Router {
-	cfg := config.NewConfigFromString(input)
+	logger := logrus.New()
-	r, err := NewRouter(cfg)
+	logger.SetOutput(ioutil.Discard)
+	cfg := config.NewConfigFromString(logger, input)
+	r, err := NewRouter(logger, cfg, bridgemap.FullMap)
 	if err != nil {
 		fmt.Println(err)
 	}
@@ -165,25 +189,33 @@ func TestNewRouter(t *testing.T) {
 	assert.Equal(t, 1, len(r.Gateways))
 	assert.Equal(t, 4, len(r.Gateways["bridge1"].Bridges))
 	assert.Equal(t, 4, len(r.Gateways["bridge1"].Channels))
-
 	r = maketestRouter(testconfig2)
 	assert.Equal(t, 2, len(r.Gateways))
 	assert.Equal(t, 4, len(r.Gateways["bridge1"].Bridges))
 	assert.Equal(t, 3, len(r.Gateways["bridge2"].Bridges))
 	assert.Equal(t, 4, len(r.Gateways["bridge1"].Channels))
 	assert.Equal(t, 3, len(r.Gateways["bridge2"].Channels))
-	assert.Equal(t, &config.ChannelInfo{Name: "42wim/testroom", Direction: "out",
+	assert.Equal(t, &config.ChannelInfo{
-		ID: "42wim/testroomgitter.42wim", Account: "gitter.42wim",
+		Name:        "42wim/testroom",
-		SameChannel: map[string]bool{"bridge2": false}},
+		Direction:   "out",
-		r.Gateways["bridge2"].Channels["42wim/testroomgitter.42wim"])
+		ID:          "42wim/testroomgitter.42wim",
-	assert.Equal(t, &config.ChannelInfo{Name: "42wim/testroom", Direction: "in",
+		Account:     "gitter.42wim",
-		ID: "42wim/testroomgitter.42wim", Account: "gitter.42wim",
+		SameChannel: map[string]bool{"bridge2": false},
-		SameChannel: map[string]bool{"bridge1": false}},
+	}, r.Gateways["bridge2"].Channels["42wim/testroomgitter.42wim"])
-		r.Gateways["bridge1"].Channels["42wim/testroomgitter.42wim"])
+	assert.Equal(t, &config.ChannelInfo{
-	assert.Equal(t, &config.ChannelInfo{Name: "general", Direction: "inout",
+		Name:        "42wim/testroom",
-		ID: "generaldiscord.test", Account: "discord.test",
+		Direction:   "in",
-		SameChannel: map[string]bool{"bridge1": false}},
+		ID:          "42wim/testroomgitter.42wim",
-		r.Gateways["bridge1"].Channels["generaldiscord.test"])
+		Account:     "gitter.42wim",
+		SameChannel: map[string]bool{"bridge1": false},
+	}, r.Gateways["bridge1"].Channels["42wim/testroomgitter.42wim"])
+	assert.Equal(t, &config.ChannelInfo{
+		Name:        "general",
+		Direction:   "inout",
+		ID:          "generaldiscord.test",
+		Account:     "discord.test",
+		SameChannel: map[string]bool{"bridge1": false},
+	}, r.Gateways["bridge1"].Channels["generaldiscord.test"])
 }
 
 func TestGetDestChannel(t *testing.T) {
@@ -192,11 +224,23 @@ func TestGetDestChannel(t *testing.T) {
 	for _, br := range r.Gateways["bridge1"].Bridges {
 		switch br.Account {
 		case "discord.test":
-			assert.Equal(t, []config.ChannelInfo{{Name: "general", Account: "discord.test", Direction: "inout", ID: "generaldiscord.test", SameChannel: map[string]bool{"bridge1": false}, Options: config.ChannelOptions{Key: ""}}},
+			assert.Equal(t, []config.ChannelInfo{{
-				r.Gateways["bridge1"].getDestChannel(msg, *br))
+				Name:        "general",
+				Account:     "discord.test",
+				Direction:   "inout",
+				ID:          "generaldiscord.test",
+				SameChannel: map[string]bool{"bridge1": false},
+				Options:     config.ChannelOptions{Key: ""},
+			}}, r.Gateways["bridge1"].getDestChannel(msg, *br))
 		case "slack.test":
-			assert.Equal(t, []config.ChannelInfo{{Name: "testing", Account: "slack.test", Direction: "out", ID: "testingslack.test", SameChannel: map[string]bool{"bridge1": false}, Options: config.ChannelOptions{Key: ""}}},
+			assert.Equal(t, []config.ChannelInfo{{
-				r.Gateways["bridge1"].getDestChannel(msg, *br))
+				Name:        "testing",
+				Account:     "slack.test",
+				Direction:   "out",
+				ID:          "testingslack.test",
+				SameChannel: map[string]bool{"bridge1": false},
+				Options:     config.ChannelOptions{Key: ""},
+			}}, r.Gateways["bridge1"].getDestChannel(msg, *br))
 		case "gitter.42wim":
 			assert.Equal(t, []config.ChannelInfo(nil), r.Gateways["bridge1"].getDestChannel(msg, *br))
 		case "irc.freenode":
@@ -226,35 +270,87 @@ func TestGetDestChannelAdvanced(t *testing.T) {
 				}
 				switch gw.Name {
 				case "bridge":
-					if (msg.Channel == "#main" || msg.Channel == "-1111111111111" || msg.Channel == "irc") && (msg.Account == "irc.zzz" || msg.Account == "telegram.zzz" || msg.Account == "slack.zzz") {
+					if (msg.Channel == "#main" || msg.Channel == "-1111111111111" || msg.Channel == "irc") &&
+						(msg.Account == ircTestAccount || msg.Account == tgTestAccount || msg.Account == slackTestAccount) {
 						hits[gw.Name]++
 						switch br.Account {
-						case "irc.zzz":
+						case ircTestAccount:
-							assert.Equal(t, []config.ChannelInfo{{Name: "#main", Account: "irc.zzz", Direction: "inout", ID: "#mainirc.zzz", SameChannel: map[string]bool{"bridge": false}, Options: config.ChannelOptions{Key: ""}}}, channels)
+							assert.Equal(t, []config.ChannelInfo{{
-						case "telegram.zzz":
+								Name:        "#main",
-							assert.Equal(t, []config.ChannelInfo{{Name: "-1111111111111", Account: "telegram.zzz", Direction: "inout", ID: "-1111111111111telegram.zzz", SameChannel: map[string]bool{"bridge": false}, Options: config.ChannelOptions{Key: ""}}}, channels)
+								Account:     ircTestAccount,
-						case "slack.zzz":
+								Direction:   "inout",
-							assert.Equal(t, []config.ChannelInfo{{Name: "irc", Account: "slack.zzz", Direction: "inout", ID: "ircslack.zzz", SameChannel: map[string]bool{"bridge": false}, Options: config.ChannelOptions{Key: ""}}}, channels)
+								ID:          "#mainirc.zzz",
+								SameChannel: map[string]bool{"bridge": false},
+								Options:     config.ChannelOptions{Key: ""},
+							}}, channels)
+						case tgTestAccount:
+							assert.Equal(t, []config.ChannelInfo{{
+								Name:        "-1111111111111",
+								Account:     tgTestAccount,
+								Direction:   "inout",
+								ID:          "-1111111111111telegram.zzz",
+								SameChannel: map[string]bool{"bridge": false},
+								Options:     config.ChannelOptions{Key: ""},
+							}}, channels)
+						case slackTestAccount:
+							assert.Equal(t, []config.ChannelInfo{{
+								Name:        "irc",
+								Account:     slackTestAccount,
+								Direction:   "inout",
+								ID:          "ircslack.zzz",
+								SameChannel: map[string]bool{"bridge": false},
+								Options:     config.ChannelOptions{Key: ""},
+							}}, channels)
 						}
 					}
 				case "bridge2":
-					if (msg.Channel == "#main-help" || msg.Channel == "--444444444444") && (msg.Account == "irc.zzz" || msg.Account == "telegram.zzz") {
+					if (msg.Channel == "#main-help" || msg.Channel == "--444444444444") &&
+						(msg.Account == ircTestAccount || msg.Account == tgTestAccount) {
 						hits[gw.Name]++
 						switch br.Account {
-						case "irc.zzz":
+						case ircTestAccount:
-							assert.Equal(t, []config.ChannelInfo{{Name: "#main-help", Account: "irc.zzz", Direction: "inout", ID: "#main-helpirc.zzz", SameChannel: map[string]bool{"bridge2": false}, Options: config.ChannelOptions{Key: ""}}}, channels)
+							assert.Equal(t, []config.ChannelInfo{{
-						case "telegram.zzz":
+								Name:        "#main-help",
-							assert.Equal(t, []config.ChannelInfo{{Name: "--444444444444", Account: "telegram.zzz", Direction: "inout", ID: "--444444444444telegram.zzz", SameChannel: map[string]bool{"bridge2": false}, Options: config.ChannelOptions{Key: ""}}}, channels)
+								Account:     ircTestAccount,
+								Direction:   "inout",
+								ID:          "#main-helpirc.zzz",
+								SameChannel: map[string]bool{"bridge2": false},
+								Options:     config.ChannelOptions{Key: ""},
+							}}, channels)
+						case tgTestAccount:
+							assert.Equal(t, []config.ChannelInfo{{
+								Name:        "--444444444444",
+								Account:     tgTestAccount,
+								Direction:   "inout",
+								ID:          "--444444444444telegram.zzz",
+								SameChannel: map[string]bool{"bridge2": false},
+								Options:     config.ChannelOptions{Key: ""},
+							}}, channels)
 						}
 					}
 				case "bridge3":
-					if (msg.Channel == "#main-telegram" || msg.Channel == "--333333333333") && (msg.Account == "irc.zzz" || msg.Account == "telegram.zzz") {
+					if (msg.Channel == "#main-telegram" || msg.Channel == "--333333333333") &&
+						(msg.Account == ircTestAccount || msg.Account == tgTestAccount) {
 						hits[gw.Name]++
 						switch br.Account {
-						case "irc.zzz":
+						case ircTestAccount:
-							assert.Equal(t, []config.ChannelInfo{{Name: "#main-telegram", Account: "irc.zzz", Direction: "inout", ID: "#main-telegramirc.zzz", SameChannel: map[string]bool{"bridge3": false}, Options: config.ChannelOptions{Key: ""}}}, channels)
+							assert.Equal(t, []config.ChannelInfo{{
-						case "telegram.zzz":
+								Name:        "#main-telegram",
-							assert.Equal(t, []config.ChannelInfo{{Name: "--333333333333", Account: "telegram.zzz", Direction: "inout", ID: "--333333333333telegram.zzz", SameChannel: map[string]bool{"bridge3": false}, Options: config.ChannelOptions{Key: ""}}}, channels)
+								Account:     ircTestAccount,
+								Direction:   "inout",
+								ID:          "#main-telegramirc.zzz",
+								SameChannel: map[string]bool{"bridge3": false},
+								Options:     config.ChannelOptions{Key: ""},
+							}}, channels)
+						case tgTestAccount:
+							assert.Equal(t, []config.ChannelInfo{{
+								Name:        "--333333333333",
+								Account:     tgTestAccount,
+								Direction:   "inout",
+								ID:          "--333333333333telegram.zzz",
+								SameChannel: map[string]bool{"bridge3": false},
+								Options:     config.ChannelOptions{Key: ""},
+							}}, channels)
 						}
 					}
 				case "announcements":
@@ -264,12 +360,42 @@ func TestGetDestChannelAdvanced(t *testing.T) {
 					}
 					hits[gw.Name]++
 					switch br.Account {
-					case "irc.zzz":
+					case ircTestAccount:
-						assert.Equal(t, []config.ChannelInfo{{Name: "#main", Account: "irc.zzz", Direction: "out", ID: "#mainirc.zzz", SameChannel: map[string]bool{"announcements": false}, Options: config.ChannelOptions{Key: ""}}, {Name: "#main-help", Account: "irc.zzz", Direction: "out", ID: "#main-helpirc.zzz", SameChannel: map[string]bool{"announcements": false}, Options: config.ChannelOptions{Key: ""}}}, channels)
+						assert.Len(t, channels, 2)
-					case "slack.zzz":
+						assert.Contains(t, channels, config.ChannelInfo{
-						assert.Equal(t, []config.ChannelInfo{{Name: "general", Account: "slack.zzz", Direction: "out", ID: "generalslack.zzz", SameChannel: map[string]bool{"announcements": false}, Options: config.ChannelOptions{Key: ""}}}, channels)
+							Name:        "#main",
-					case "telegram.zzz":
+							Account:     ircTestAccount,
-						assert.Equal(t, []config.ChannelInfo{{Name: "--333333333333", Account: "telegram.zzz", Direction: "out", ID: "--333333333333telegram.zzz", SameChannel: map[string]bool{"announcements": false}, Options: config.ChannelOptions{Key: ""}}}, channels)
+							Direction:   "out",
+							ID:          "#mainirc.zzz",
+							SameChannel: map[string]bool{"announcements": false},
+							Options:     config.ChannelOptions{Key: ""},
+						})
+						assert.Contains(t, channels, config.ChannelInfo{
+							Name:        "#main-help",
+							Account:     ircTestAccount,
+							Direction:   "out",
+							ID:          "#main-helpirc.zzz",
+							SameChannel: map[string]bool{"announcements": false},
+							Options:     config.ChannelOptions{Key: ""},
+						})
+					case slackTestAccount:
+						assert.Equal(t, []config.ChannelInfo{{
+							Name:        "general",
+							Account:     slackTestAccount,
+							Direction:   "out",
+							ID:          "generalslack.zzz",
+							SameChannel: map[string]bool{"announcements": false},
+							Options:     config.ChannelOptions{Key: ""},
+						}}, channels)
+					case tgTestAccount:
+						assert.Equal(t, []config.ChannelInfo{{
+							Name:        "--333333333333",
+							Account:     tgTestAccount,
+							Direction:   "out",
+							ID:          "--333333333333telegram.zzz",
+							SameChannel: map[string]bool{"announcements": false},
+							Options:     config.ChannelOptions{Key: ""},
+						}}, channels)
 					}
 				}
 			}
@@ -277,3 +403,139 @@ func TestGetDestChannelAdvanced(t *testing.T) {
 	}
 	assert.Equal(t, map[string]int{"bridge3": 4, "bridge": 9, "announcements": 3, "bridge2": 4}, hits)
 }
+
+type ignoreTestSuite struct {
+	suite.Suite
+
+	gw *Gateway
+}
+
+func TestIgnoreSuite(t *testing.T) {
+	s := &ignoreTestSuite{}
+	suite.Run(t, s)
+}
+
+func (s *ignoreTestSuite) SetupSuite() {
+	logger := logrus.New()
+	logger.SetOutput(ioutil.Discard)
+	s.gw = &Gateway{logger: logrus.NewEntry(logger)}
+}
+func (s *ignoreTestSuite) TestIgnoreTextEmpty() {
+	extraFile := make(map[string][]interface{})
+	extraAttach := make(map[string][]interface{})
+	extraFailure := make(map[string][]interface{})
+	extraFile["file"] = append(extraFile["file"], config.FileInfo{})
+	extraAttach["attachments"] = append(extraAttach["attachments"], []string{})
+	extraFailure[config.EventFileFailureSize] = append(extraFailure[config.EventFileFailureSize], config.FileInfo{})
+
+	msgTests := map[string]struct {
+		input  *config.Message
+		output bool
+	}{
+		"usertyping": {
+			input:  &config.Message{Event: config.EventUserTyping},
+			output: false,
+		},
+		"file attach": {
+			input:  &config.Message{Extra: extraFile},
+			output: false,
+		},
+		"attachments": {
+			input:  &config.Message{Extra: extraAttach},
+			output: false,
+		},
+		config.EventFileFailureSize: {
+			input:  &config.Message{Extra: extraFailure},
+			output: false,
+		},
+		"nil extra": {
+			input:  &config.Message{Extra: nil},
+			output: true,
+		},
+		"empty": {
+			input:  &config.Message{},
+			output: true,
+		},
+	}
+	for testname, testcase := range msgTests {
+		output := s.gw.ignoreTextEmpty(testcase.input)
+		s.Assert().Equalf(testcase.output, output, "case '%s' failed", testname)
+	}
+
+}
+
+func (s *ignoreTestSuite) TestIgnoreTexts() {
+	msgTests := map[string]struct {
+		input  string
+		re     []string
+		output bool
+	}{
+		"no regex": {
+			input:  "a text message",
+			re:     []string{},
+			output: false,
+		},
+		"simple regex": {
+			input:  "a text message",
+			re:     []string{"text"},
+			output: true,
+		},
+		"multiple regex fail": {
+			input:  "a text message",
+			re:     []string{"abc", "123$"},
+			output: false,
+		},
+		"multiple regex pass": {
+			input:  "a text message",
+			re:     []string{"lala", "sage$"},
+			output: true,
+		},
+	}
+	for testname, testcase := range msgTests {
+		output := s.gw.ignoreText(testcase.input, testcase.re)
+		s.Assert().Equalf(testcase.output, output, "case '%s' failed", testname)
+	}
+}
+
+func (s *ignoreTestSuite) TestIgnoreNicks() {
+	msgTests := map[string]struct {
+		input  string
+		re     []string
+		output bool
+	}{
+		"no entry": {
+			input:  "user",
+			re:     []string{},
+			output: false,
+		},
+		"one entry": {
+			input:  "user",
+			re:     []string{"user"},
+			output: true,
+		},
+		"multiple entries": {
+			input:  "user",
+			re:     []string{"abc", "user"},
+			output: true,
+		},
+		"multiple entries fail": {
+			input:  "user",
+			re:     []string{"abc", "def"},
+			output: false,
+		},
+	}
+	for testname, testcase := range msgTests {
+		output := s.gw.ignoreText(testcase.input, testcase.re)
+		s.Assert().Equalf(testcase.output, output, "case '%s' failed", testname)
+	}
+}
+
+func BenchmarkTengo(b *testing.B) {
+	msg := &config.Message{Username: "user", Text: "blah testing", Account: "protocol.account", Channel: "mychannel"}
+	for n := 0; n < b.N; n++ {
+		err := modifyMessageTengo("bench.tengo", msg)
+		if err != nil {
+			return
+		}
+	}
+}

gateway/handlers.go

@@ -0,0 +1,275 @@
+package gateway
+
+import (
+	"bytes"
+	"crypto/sha1" //nolint:gosec
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/42wim/matterbridge/bridge"
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/42wim/matterbridge/gateway/bridgemap"
+)
+
+// handleEventFailure handles failures and reconnects bridges.
+func (r *Router) handleEventFailure(msg *config.Message) {
+	if msg.Event != config.EventFailure {
+		return
+	}
+	for _, gw := range r.Gateways {
+		for _, br := range gw.Bridges {
+			if msg.Account == br.Account {
+				go gw.reconnectBridge(br)
+				return
+			}
+		}
+	}
+}
+
+// handleEventGetChannelMembers handles channel members
+func (r *Router) handleEventGetChannelMembers(msg *config.Message) {
+	if msg.Event != config.EventGetChannelMembers {
+		return
+	}
+	for _, gw := range r.Gateways {
+		for _, br := range gw.Bridges {
+			if msg.Account == br.Account {
+				cMembers := msg.Extra[config.EventGetChannelMembers][0].(config.ChannelMembers)
+				r.logger.Debugf("Syncing channelmembers from %s", msg.Account)
+				br.SetChannelMembers(&cMembers)
+				return
+			}
+		}
+	}
+}
+
+// handleEventRejoinChannels handles rejoining of channels.
+func (r *Router) handleEventRejoinChannels(msg *config.Message) {
+	if msg.Event != config.EventRejoinChannels {
+		return
+	}
+	for _, gw := range r.Gateways {
+		for _, br := range gw.Bridges {
+			if msg.Account == br.Account {
+				br.Joined = make(map[string]bool)
+				if err := br.JoinChannels(); err != nil {
+					r.logger.Errorf("channel join failed for %s: %s", msg.Account, err)
+				}
+			}
+		}
+	}
+}
+
+// handleFiles uploads or places all files on the given msg to the MediaServer and
+// adds the new URL of the file on the MediaServer onto the given msg.
+func (gw *Gateway) handleFiles(msg *config.Message) {
+	reg := regexp.MustCompile("[^a-zA-Z0-9]+")
+
+	// If we don't have a attachfield or we don't have a mediaserver configured return
+	if msg.Extra == nil ||
+		(gw.BridgeValues().General.MediaServerUpload == "" &&
+			gw.BridgeValues().General.MediaDownloadPath == "") {
+		return
+	}
+
+	// If we don't have files, nothing to upload.
+	if len(msg.Extra["file"]) == 0 {
+		return
+	}
+
+	for i, f := range msg.Extra["file"] {
+		fi := f.(config.FileInfo)
+		ext := filepath.Ext(fi.Name)
+		fi.Name = fi.Name[0 : len(fi.Name)-len(ext)]
+		fi.Name = reg.ReplaceAllString(fi.Name, "_")
+		fi.Name += ext
+
+		sha1sum := fmt.Sprintf("%x", sha1.Sum(*fi.Data))[:8] //nolint:gosec
+
+		if gw.BridgeValues().General.MediaServerUpload != "" {
+			// Use MediaServerUpload. Upload using a PUT HTTP request and basicauth.
+			if err := gw.handleFilesUpload(&fi); err != nil {
+				gw.logger.Error(err)
+				continue
+			}
+		} else {
+			// Use MediaServerPath. Place the file on the current filesystem.
+			if err := gw.handleFilesLocal(&fi); err != nil {
+				gw.logger.Error(err)
+				continue
+			}
+		}
+
+		// Download URL.
+		durl := gw.BridgeValues().General.MediaServerDownload + "/" + sha1sum + "/" + fi.Name
+
+		gw.logger.Debugf("mediaserver download URL = %s", durl)
+
+		// We uploaded/placed the file successfully. Add the SHA and URL.
+		extra := msg.Extra["file"][i].(config.FileInfo)
+		extra.URL = durl
+		extra.SHA = sha1sum
+		msg.Extra["file"][i] = extra
+	}
+}
+
+// handleFilesUpload uses MediaServerUpload configuration to upload the file.
+// Returns error on failure.
+func (gw *Gateway) handleFilesUpload(fi *config.FileInfo) error {
+	client := &http.Client{
+		Timeout: time.Second * 5,
+	}
+	// Use MediaServerUpload. Upload using a PUT HTTP request and basicauth.
+	sha1sum := fmt.Sprintf("%x", sha1.Sum(*fi.Data))[:8] //nolint:gosec
+	url := gw.BridgeValues().General.MediaServerUpload + "/" + sha1sum + "/" + fi.Name
+
+	req, err := http.NewRequest("PUT", url, bytes.NewReader(*fi.Data))
+	if err != nil {
+		return fmt.Errorf("mediaserver upload failed, could not create request: %#v", err)
+	}
+
+	gw.logger.Debugf("mediaserver upload url: %s", url)
+
+	req.Header.Set("Content-Type", "binary/octet-stream")
+	_, err = client.Do(req)
+	if err != nil {
+		return fmt.Errorf("mediaserver upload failed, could not Do request: %#v", err)
+	}
+	return nil
+}
+
+// handleFilesLocal use MediaServerPath configuration, places the file on the current filesystem.
+// Returns error on failure.
+func (gw *Gateway) handleFilesLocal(fi *config.FileInfo) error {
+	sha1sum := fmt.Sprintf("%x", sha1.Sum(*fi.Data))[:8] //nolint:gosec
+	dir := gw.BridgeValues().General.MediaDownloadPath + "/" + sha1sum
+	err := os.Mkdir(dir, os.ModePerm)
+	if err != nil && !os.IsExist(err) {
+		return fmt.Errorf("mediaserver path failed, could not mkdir: %s %#v", err, err)
+	}
+
+	path := dir + "/" + fi.Name
+	gw.logger.Debugf("mediaserver path placing file: %s", path)
+
+	err = ioutil.WriteFile(path, *fi.Data, os.ModePerm)
+	if err != nil {
+		return fmt.Errorf("mediaserver path failed, could not writefile: %s %#v", err, err)
+	}
+	return nil
+}
+
+// ignoreEvent returns true if we need to ignore this event for the specified destination bridge.
+func (gw *Gateway) ignoreEvent(event string, dest *bridge.Bridge) bool {
+	switch event {
+	case config.EventAvatarDownload:
+		// Avatar downloads are only relevant for telegram and mattermost for now
+		if dest.Protocol != "mattermost" && dest.Protocol != "telegram" && dest.Protocol != "xmpp" {
+			return true
+		}
+	case config.EventJoinLeave:
+		// only relay join/part when configured
+		if !dest.GetBool("ShowJoinPart") {
+			return true
+		}
+	case config.EventTopicChange:
+		// only relay topic change when used in some way on other side
+		if !dest.GetBool("ShowTopicChange") && !dest.GetBool("SyncTopic") {
+			return true
+		}
+	}
+	return false
+}
+
+// handleMessage makes sure the message get sent to the correct bridge/channels.
+// Returns an array of msg ID's
+func (gw *Gateway) handleMessage(rmsg *config.Message, dest *bridge.Bridge) []*BrMsgID {
+	var brMsgIDs []*BrMsgID
+
+	// Not all bridges support "user is typing" indications so skip the message
+	// if the targeted bridge does not support it.
+	if rmsg.Event == config.EventUserTyping {
+		if _, ok := bridgemap.UserTypingSupport[dest.Protocol]; !ok {
+			return nil
+		}
+	}
+
+	// if we have an attached file, or other info
+	if rmsg.Extra != nil && len(rmsg.Extra[config.EventFileFailureSize]) != 0 && rmsg.Text == "" {
+		return brMsgIDs
+	}
+
+	if gw.ignoreEvent(rmsg.Event, dest) {
+		return brMsgIDs
+	}
+
+	// broadcast to every out channel (irc QUIT)
+	if rmsg.Channel == "" && rmsg.Event != config.EventJoinLeave {
+		gw.logger.Debug("empty channel")
+		return brMsgIDs
+	}
+
+	// Get the ID of the parent message in thread
+	var canonicalParentMsgID string
+	if rmsg.ParentID != "" && dest.GetBool("PreserveThreading") {
+		canonicalParentMsgID = gw.FindCanonicalMsgID(rmsg.Protocol, rmsg.ParentID)
+	}
+
+	channels := gw.getDestChannel(rmsg, *dest)
+	for idx := range channels {
+		channel := &channels[idx]
+		msgID, err := gw.SendMessage(rmsg, dest, channel, canonicalParentMsgID)
+		if err != nil {
+			gw.logger.Errorf("SendMessage failed: %s", err)
+			continue
+		}
+		if msgID == "" {
+			continue
+		}
+		brMsgIDs = append(brMsgIDs, &BrMsgID{dest, dest.Protocol + " " + msgID, channel.ID})
+	}
+	return brMsgIDs
+}
+
+func (gw *Gateway) handleExtractNicks(msg *config.Message) {
+	var err error
+	br := gw.Bridges[msg.Account]
+	for _, outer := range br.GetStringSlice2D("ExtractNicks") {
+		search := outer[0]
+		replace := outer[1]
+		msg.Username, msg.Text, err = extractNick(search, replace, msg.Username, msg.Text)
+		if err != nil {
+			gw.logger.Errorf("regexp in %s failed: %s", msg.Account, err)
+			break
+		}
+	}
+}
+
+// extractNick searches for a username (based on "search" a regular expression).
+// if this matches it extracts a nick (based on "extract" another regular expression) from text
+// and replaces username with this result.
+// returns error if the regexp doesn't compile.
+func extractNick(search, extract, username, text string) (string, string, error) {
+	re, err := regexp.Compile(search)
+	if err != nil {
+		return username, text, err
+	}
+	if re.MatchString(username) {
+		re, err = regexp.Compile(extract)
+		if err != nil {
+			return username, text, err
+		}
+		res := re.FindAllStringSubmatch(text, 1)
+		// only replace if we have exactly 1 match
+		if len(res) > 0 && len(res[0]) == 2 {
+			username = res[0][1]
+			text = strings.Replace(text, res[0][0], "", 1)
+		}
+	}
+	return username, text, nil
+}

gateway/handlers_test.go

@@ -0,0 +1,75 @@
+package gateway
+
+import (
+	"github.com/42wim/matterbridge/bridge"
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/stretchr/testify/assert"
+
+	"testing"
+)
+
+func TestIgnoreEvent(t *testing.T) {
+	eventTests := map[string]struct {
+		input  string
+		dest   *bridge.Bridge
+		output bool
+	}{
+		"avatar mattermost": {
+			input:  config.EventAvatarDownload,
+			dest:   &bridge.Bridge{Protocol: "mattermost"},
+			output: false,
+		},
+		"avatar slack": {
+			input:  config.EventAvatarDownload,
+			dest:   &bridge.Bridge{Protocol: "slack"},
+			output: true,
+		},
+		"avatar telegram": {
+			input:  config.EventAvatarDownload,
+			dest:   &bridge.Bridge{Protocol: "telegram"},
+			output: false,
+		},
+	}
+	gw := &Gateway{}
+	for testname, testcase := range eventTests {
+		output := gw.ignoreEvent(testcase.input, testcase.dest)
+		assert.Equalf(t, testcase.output, output, "case '%s' failed", testname)
+	}
+
+}
+
+func TestExtractNick(t *testing.T) {
+	eventTests := map[string]struct {
+		search         string
+		extract        string
+		username       string
+		text           string
+		resultUsername string
+		resultText     string
+	}{
+		"test1": {
+			search:         "fromgitter",
+			extract:        "<(.*?)>\\s+",
+			username:       "fromgitter",
+			text:           "<userx> blahblah",
+			resultUsername: "userx",
+			resultText:     "blahblah",
+		},
+		"test2": {
+			search: "<.*?bot>",
+			//extract:        `\((.*?)\)\s+`,
+			extract:        "\\((.*?)\\)\\s+",
+			username:       "<matterbot>",
+			text:           "(userx) blahblah (abc) test",
+			resultUsername: "userx",
+			resultText:     "blahblah (abc) test",
+		},
+	}
+	//	gw := &Gateway{}
+	for testname, testcase := range eventTests {
+		resultUsername, resultText, _ := extractNick(testcase.search, testcase.extract, testcase.username, testcase.text)
+		assert.Equalf(t, testcase.resultUsername, resultUsername, "case '%s' failed", testname)
+		assert.Equalf(t, testcase.resultText, resultText, "case '%s' failed", testname)
+	}
+
+}

gateway/router.go

@@ -2,26 +2,45 @@ package gateway
 
 import (
 	"fmt"
+	"sync"
+	"time"
 
 	"github.com/42wim/matterbridge/bridge"
 	"github.com/42wim/matterbridge/bridge/config"
-	samechannelgateway "github.com/42wim/matterbridge/gateway/samechannel"
+	"github.com/42wim/matterbridge/gateway/samechannel"
-	//	"github.com/davecgh/go-spew/spew"
+	"github.com/sirupsen/logrus"
-	"time"
 )
 
 type Router struct {
-	Gateways map[string]*Gateway
+	config.Config
-	Message  chan config.Message
+	sync.RWMutex
-	*config.Config
+
+	BridgeMap        map[string]bridge.Factory
+	Gateways         map[string]*Gateway
+	Message          chan config.Message
+	MattermostPlugin chan config.Message
+
+	logger *logrus.Entry
 }
 
-func NewRouter(cfg *config.Config) (*Router, error) {
+// NewRouter initializes a new Matterbridge router for the specified configuration and
-	r := &Router{Message: make(chan config.Message), Gateways: make(map[string]*Gateway), Config: cfg}
+// sets up all required gateways.
-	sgw := samechannelgateway.New(cfg)
+func NewRouter(rootLogger *logrus.Logger, cfg config.Config, bridgeMap map[string]bridge.Factory) (*Router, error) {
-	gwconfigs := sgw.GetConfig()
+	logger := rootLogger.WithFields(logrus.Fields{"prefix": "router"})
 
-	for _, entry := range append(gwconfigs, cfg.Gateway...) {
+	r := &Router{
+		Config:           cfg,
+		BridgeMap:        bridgeMap,
+		Message:          make(chan config.Message),
+		MattermostPlugin: make(chan config.Message),
+		Gateways:         make(map[string]*Gateway),
+		logger:           logger,
+	}
+	sgw := samechannel.New(cfg)
+	gwconfigs := append(sgw.GetConfig(), cfg.BridgeValues().Gateway...)
+
+	for idx := range gwconfigs {
+		entry := &gwconfigs[idx]
 		if !entry.Enable {
 			continue
 		}
@@ -31,34 +50,72 @@ func NewRouter(cfg *config.Config) (*Router, error) {
 		if _, ok := r.Gateways[entry.Name]; ok {
 			return nil, fmt.Errorf("Gateway with name %s already exists", entry.Name)
 		}
-		r.Gateways[entry.Name] = New(entry, r)
+		r.Gateways[entry.Name] = New(rootLogger, entry, r)
 	}
 	return r, nil
 }
 
+// Start will connect all gateways belonging to this router and subsequently route messages
+// between them.
 func (r *Router) Start() error {
 	m := make(map[string]*bridge.Bridge)
+	if len(r.Gateways) == 0 {
+		return fmt.Errorf("no [[gateway]] configured. See https://github.com/42wim/matterbridge/wiki/How-to-create-your-config for more info")
+	}
 	for _, gw := range r.Gateways {
-		flog.Infof("Parsing gateway %s", gw.Name)
+		r.logger.Infof("Parsing gateway %s", gw.Name)
+		if len(gw.Bridges) == 0 {
+			return fmt.Errorf("no bridges configured for gateway %s. See https://github.com/42wim/matterbridge/wiki/How-to-create-your-config for more info", gw.Name)
+		}
 		for _, br := range gw.Bridges {
 			m[br.Account] = br
 		}
 	}
 	for _, br := range m {
-		flog.Infof("Starting bridge: %s ", br.Account)
+		r.logger.Infof("Starting bridge: %s ", br.Account)
 		err := br.Connect()
 		if err != nil {
-			return fmt.Errorf("Bridge %s failed to start: %v", br.Account, err)
+			e := fmt.Errorf("Bridge %s failed to start: %v", br.Account, err)
+			if r.disableBridge(br, e) {
+				continue
+			}
+			return e
 		}
 		err = br.JoinChannels()
 		if err != nil {
-			return fmt.Errorf("Bridge %s failed to join channel: %v", br.Account, err)
+			e := fmt.Errorf("Bridge %s failed to join channel: %v", br.Account, err)
+			if r.disableBridge(br, e) {
+				continue
+			}
+			return e
+		}
+	}
+	// remove unused bridges
+	for _, gw := range r.Gateways {
+		for i, br := range gw.Bridges {
+			if br.Bridger == nil {
+				r.logger.Errorf("removing failed bridge %s", i)
+				delete(gw.Bridges, i)
+			}
 		}
 	}
 	go r.handleReceive()
+	//go r.updateChannelMembers()
 	return nil
 }
 
+// disableBridge returns true and empties a bridge if we have IgnoreFailureOnStart configured
+// otherwise returns false
+func (r *Router) disableBridge(br *bridge.Bridge, err error) bool {
+	if r.BridgeValues().General.IgnoreFailureOnStart {
+		r.logger.Error(err)
+		// setting this bridge empty
+		*br = bridge.Bridge{}
+		return true
+	}
+	return false
+}
+
 func (r *Router) getBridge(account string) *bridge.Bridge {
 	for _, gw := range r.Gateways {
 		if br, ok := gw.Bridges[account]; ok {
@@ -70,42 +127,65 @@ func (r *Router) getBridge(account string) *bridge.Bridge {
 
 func (r *Router) handleReceive() {
 	for msg := range r.Message {
-		if msg.Event == config.EVENT_FAILURE {
+		msg := msg // scopelint
-		Loop:
+		r.handleEventGetChannelMembers(&msg)
-			for _, gw := range r.Gateways {
+		r.handleEventFailure(&msg)
-				for _, br := range gw.Bridges {
+		r.handleEventRejoinChannels(&msg)
-					if msg.Account == br.Account {
+
-						go gw.reconnectBridge(br)
+		// Set message protocol based on the account it came from
-						break Loop
+		msg.Protocol = r.getBridge(msg.Account).Protocol
-					}
+
-				}
+		filesHandled := false
-			}
-		}
-		if msg.Event == config.EVENT_REJOIN_CHANNELS {
-			for _, gw := range r.Gateways {
-				for _, br := range gw.Bridges {
-					if msg.Account == br.Account {
-						br.Joined = make(map[string]bool)
-						br.JoinChannels()
-					}
-				}
-			}
-		}
 		for _, gw := range r.Gateways {
 			// record all the message ID's of the different bridges
 			var msgIDs []*BrMsgID
-			if !gw.ignoreMessage(&msg) {
+			if gw.ignoreMessage(&msg) {
-				msg.Timestamp = time.Now()
+				continue
-				gw.modifyMessage(&msg)
+			}
+			msg.Timestamp = time.Now()
+			gw.modifyMessage(&msg)
+			if !filesHandled {
 				gw.handleFiles(&msg)
-				for _, br := range gw.Bridges {
+				filesHandled = true
-					msgIDs = append(msgIDs, gw.handleMessage(msg, br)...)
+			}
-				}
+			for _, br := range gw.Bridges {
-				// only add the message ID if it doesn't already exists
+				msgIDs = append(msgIDs, gw.handleMessage(&msg, br)...)
-				if _, ok := gw.Messages.Get(msg.ID); !ok && msg.ID != "" {
+			}
-					gw.Messages.Add(msg.ID, msgIDs)
+
+			if msg.ID != "" {
+				_, exists := gw.Messages.Get(msg.Protocol + " " + msg.ID)
+
+				// Only add the message ID if it doesn't already exist
+				//
+				// For some bridges we always add/update the message ID.
+				// This is necessary as msgIDs will change if a bridge returns
+				// a different ID in response to edits.
+				if !exists || msg.Protocol == "discord" {
+					gw.Messages.Add(msg.Protocol+" "+msg.ID, msgIDs)
 				}
 			}
 		}
 	}
 }
+
+// updateChannelMembers sends every minute an GetChannelMembers event to all bridges.
+func (r *Router) updateChannelMembers() {
+	// TODO sleep a minute because slack can take a while
+	// fix this by having actually connectionDone events send to the router
+	time.Sleep(time.Minute)
+	for {
+		for _, gw := range r.Gateways {
+			for _, br := range gw.Bridges {
+				// only for slack now
+				if br.Protocol != "slack" {
+					continue
+				}
+				r.logger.Debugf("sending %s to %s", config.EventGetChannelMembers, br.Account)
+				if _, err := br.Send(config.Message{Event: config.EventGetChannelMembers}); err != nil {
+					r.logger.Errorf("updateChannelMembers: %s", err)
+				}
+			}
+		}
+		time.Sleep(time.Minute)
+	}
+}

gateway/samechannel/samechannel.go

@@ -1,21 +1,21 @@
-package samechannelgateway
+package samechannel
 
 import (
 	"github.com/42wim/matterbridge/bridge/config"
 )
 
 type SameChannelGateway struct {
-	*config.Config
+	config.Config
 }
 
-func New(cfg *config.Config) *SameChannelGateway {
+func New(cfg config.Config) *SameChannelGateway {
 	return &SameChannelGateway{Config: cfg}
 }
 
 func (sgw *SameChannelGateway) GetConfig() []config.Gateway {
 	var gwconfigs []config.Gateway
 	cfg := sgw.Config
-	for _, gw := range cfg.SameChannelGateway {
+	for _, gw := range cfg.BridgeValues().SameChannelGateway {
 		gwconfig := config.Gateway{Name: gw.Name, Enable: gw.Enable}
 		for _, account := range gw.Accounts {
 			for _, channel := range gw.Channels {

gateway/samechannel/samechannel_test.go

@@ -1,16 +1,15 @@
-package samechannelgateway
+package samechannel
 
 import (
-	"fmt"
+	"io/ioutil"
+	"testing"
 
 	"github.com/42wim/matterbridge/bridge/config"
-	"github.com/BurntSushi/toml"
+	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
-
-	"testing"
 )
 
-var testconfig = `
+const testConfig = `
 [mattermost.test]
 [slack.test]
 
@@ -21,12 +20,58 @@ var testconfig = `
       channels = [ "testing","testing2","testing10"]
 `
 
-func TestGetConfig(t *testing.T) {
+var (
-	var cfg *config.Config
+	expectedConfig = config.Gateway{
-	if _, err := toml.Decode(testconfig, &cfg); err != nil {
+		Name:   "blah",
-		fmt.Println(err)
+		Enable: true,
+		In:     []config.Bridge(nil),
+		Out:    []config.Bridge(nil),
+		InOut: []config.Bridge{
+			{
+				Account:     "mattermost.test",
+				Channel:     "testing",
+				Options:     config.ChannelOptions{Key: ""},
+				SameChannel: true,
+			},
+			{
+				Account:     "mattermost.test",
+				Channel:     "testing2",
+				Options:     config.ChannelOptions{Key: ""},
+				SameChannel: true,
+			},
+			{
+				Account:     "mattermost.test",
+				Channel:     "testing10",
+				Options:     config.ChannelOptions{Key: ""},
+				SameChannel: true,
+			},
+			{
+				Account:     "slack.test",
+				Channel:     "testing",
+				Options:     config.ChannelOptions{Key: ""},
+				SameChannel: true,
+			},
+			{
+				Account:     "slack.test",
+				Channel:     "testing2",
+				Options:     config.ChannelOptions{Key: ""},
+				SameChannel: true,
+			},
+			{
+				Account:     "slack.test",
+				Channel:     "testing10",
+				Options:     config.ChannelOptions{Key: ""},
+				SameChannel: true,
+			},
+		},
 	}
+)
+
+func TestGetConfig(t *testing.T) {
+	logger := logrus.New()
+	logger.SetOutput(ioutil.Discard)
+	cfg := config.NewConfigFromString(logger, []byte(testConfig))
 	sgw := New(cfg)
 	configs := sgw.GetConfig()
-	assert.Equal(t, []config.Gateway{{Name: "blah", Enable: true, In: []config.Bridge(nil), Out: []config.Bridge(nil), InOut: []config.Bridge{{Account: "mattermost.test", Channel: "testing", Options: config.ChannelOptions{Key: ""}, SameChannel: true}, {Account: "mattermost.test", Channel: "testing2", Options: config.ChannelOptions{Key: ""}, SameChannel: true}, {Account: "mattermost.test", Channel: "testing10", Options: config.ChannelOptions{Key: ""}, SameChannel: true}, {Account: "slack.test", Channel: "testing", Options: config.ChannelOptions{Key: ""}, SameChannel: true}, {Account: "slack.test", Channel: "testing2", Options: config.ChannelOptions{Key: ""}, SameChannel: true}, {Account: "slack.test", Channel: "testing10", Options: config.ChannelOptions{Key: ""}, SameChannel: true}}}}, configs)
+	assert.Equal(t, []config.Gateway{expectedConfig}, configs)
 }

go.mod

@@ -0,0 +1,56 @@
+module github.com/42wim/matterbridge
+
+require (
+	github.com/42wim/go-gitter v0.0.0-20170828205020-017310c2d557
+	github.com/Baozisoftware/qrcode-terminal-go v0.0.0-20170407111555-c0650d8dff0f
+	github.com/Jeffail/gabs v1.1.1 // indirect
+	github.com/Philipp15b/go-steam v1.0.1-0.20190816133340-b04c5a83c1c0
+	github.com/Rhymen/go-whatsapp v0.1.1-0.20200818115958-f07a700b9819
+	github.com/d5/tengo/v2 v2.6.0
+	github.com/davecgh/go-spew v1.1.1
+	github.com/fsnotify/fsnotify v1.4.9
+	github.com/go-telegram-bot-api/telegram-bot-api v1.0.1-0.20200524105306-7434b0456e81
+	github.com/gomarkdown/markdown v0.0.0-20200824053859-8c8b3816f167
+	github.com/google/gops v0.3.11
+	github.com/gopackage/ddp v0.0.0-20170117053602-652027933df4 // indirect
+	github.com/gorilla/schema v1.2.0
+	github.com/gorilla/websocket v1.4.2
+	github.com/hashicorp/golang-lru v0.5.4
+	github.com/jpillora/backoff v1.0.0
+	github.com/keybase/go-keybase-chat-bot v0.0.0-20200505163032-5cacf52379da
+	github.com/labstack/echo/v4 v4.1.17
+	github.com/lrstanley/girc v0.0.0-20190801035559-4fc93959e1a7
+	github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd
+	github.com/matterbridge/Rocket.Chat.Go.SDK v0.0.0-20200411204219-d5c18ce75048
+	github.com/matterbridge/discordgo v0.22.0
+	github.com/matterbridge/emoji v2.1.1-0.20191117213217-af507f6b02db+incompatible
+	github.com/matterbridge/go-xmpp v0.0.0-20200418225040-c8a3a57b4050
+	github.com/matterbridge/gozulipbot v0.0.0-20200820220548-be5824faa913
+	github.com/matterbridge/logrus-prefixed-formatter v0.5.3-0.20200523233437-d971309a77ba
+	github.com/mattermost/mattermost-server/v5 v5.25.2
+	github.com/mattn/godown v0.0.0-20200217152941-afc959f6a561
+	github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect
+	github.com/missdeer/golib v1.0.3
+	github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474 // indirect
+	github.com/mrexodia/wray v0.0.0-20160318003008-78a2c1f284ff // indirect
+	github.com/nelsonken/gomf v0.0.0-20180504123937-a9dd2f9deae9
+	github.com/paulrosania/go-charset v0.0.0-20190326053356-55c9d7a5834c
+	github.com/rs/xid v1.2.1
+	github.com/russross/blackfriday v1.5.2
+	github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca
+	github.com/shazow/ssh-chat v1.8.3-0.20200308224626-80ddf1f43a98
+	github.com/sirupsen/logrus v1.6.0
+	github.com/slack-go/slack v0.6.6
+	github.com/spf13/viper v1.7.1
+	github.com/stretchr/testify v1.6.1
+	github.com/writeas/go-strip-markdown v2.0.1+incompatible
+	github.com/x-cray/logrus-prefixed-formatter v0.5.2 // indirect
+	github.com/yaegashi/msgraph.go v0.1.4
+	github.com/zfjagann/golang-ring v0.0.0-20190304061218-d34796e0a6c2
+	golang.org/x/image v0.0.0-20200801110659-972c09e46d76
+	golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43
+	gomod.garykim.dev/nc-talk v0.1.3
+	gopkg.in/olahol/melody.v1 v1.0.0-20170518105555-d52139073376
+)
+
+go 1.13

hook/rockethook/rockethook.go

@@ -38,7 +38,7 @@ type Config struct {
 func New(url string, config Config) *Client {
 	c := &Client{In: make(chan Message), Config: config}
 	tr := &http.Transport{
-		TLSClientConfig: &tls.Config{InsecureSkipVerify: config.InsecureSkipVerify},
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: config.InsecureSkipVerify}, //nolint:gosec
 	}
 	c.httpclient = &http.Client{Transport: tr}
 	_, _, err := net.SplitHostPort(c.BindAddress)