Release v1.18.3 (#1229)

Signed-off-by: Tilo Spannagel <development@tilosp.de>

.dockerignore

@@ -0,0 +1,2 @@
+Dockerfile
+tgs.Dockerfile

.github/workflows/development.yml

@@ -0,0 +1,57 @@
+name: Development
+on: [push, pull_request]
+jobs:
+  lint:
+    name: golangci-lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 20
+      - name: Run golangci-lint
+        uses: golangci/golangci-lint-action@v2
+        with:
+          version: v1.29
+          args: "-v --new-from-rev HEAD~5"
+  test-build-upload:
+    strategy:
+      matrix:
+        go-version: [1.14.x, 1.15.x]
+        platform: [ubuntu-latest]
+    runs-on: ${{ matrix.platform }}
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+    - name: Checkout code
+      uses: actions/checkout@v2
+      with:
+          fetch-depth: 0
+    - name: Test
+      run: go test ./... -mod=vendor
+    - name: Build
+      run: |
+        mkdir -p output/{win,lin,arm,mac}
+        VERSION=$(git describe --tags)
+        GOOS=linux GOARCH=amd64 go build -mod=vendor -ldflags "-s -X main.githash=$(git log --pretty=format:'%h' -n 1)" -o output/lin/matterbridge-$VERSION-linux-amd64
+        GOOS=windows GOARCH=amd64 go build -mod=vendor -ldflags "-s -X main.githash=$(git log --pretty=format:'%h' -n 1)" -o output/win/matterbridge-$VERSION-windows-amd64.exe
+        GOOS=darwin GOARCH=amd64 go build -mod=vendor -ldflags "-s -X main.githash=$(git log --pretty=format:'%h' -n 1)" -o output/mac/matterbridge-$VERSION-darwin-amd64
+    - name: Upload linux 64-bit
+      if: startsWith(matrix.go-version,'1.15')
+      uses: actions/upload-artifact@v2
+      with:
+        name: matterbridge-linux-64bit
+        path: output/lin
+    - name: Upload windows 64-bit
+      if: startsWith(matrix.go-version,'1.15')
+      uses: actions/upload-artifact@v2
+      with:
+        name: matterbridge-windows-64bit
+        path: output/win
+    - name: Upload darwin 64-bit
+      if: startsWith(matrix.go-version,'1.15')
+      uses: actions/upload-artifact@v2
+      with:
+        name: matterbridge-darwin-64bit
+        path: output/mac

.golangci.yaml

@@ -176,7 +176,13 @@ linters:
     - prealloc
     - wsl
     - gomnd
-
+    - godox
+    - goerr113
+    - testpackage
+    - godot
+    - interfacer
+    - goheader
+    - noctx
 
 # rules to deal with reported isues
 issues:

.travis.yml

@@ -1,56 +0,0 @@
-language: go
-go_import_path: github.com/42wim/matterbridge
-
-# We have everything vendored so this helps TravisCI not run `go get ...`.
-install: true
-
-git:
-  depth: 200
-
-notifications:
-  email: false
-
-branches:
-  only:
-  - master
-  - /.*/
-
-jobs:
-  include:
-  - stage: lint
-    # Run linting in one Go environment only.
-    script: ./ci/lint.sh
-    go: 1.14.x
-    env:
-    - GO111MODULE=on
-    - GOLANGCI_VERSION="v1.23.7"
-  - stage: test
-    # Run tests in a combination of Go environments.
-    script: ./ci/test.sh
-    go: 1.13.x
-    env:
-    - GOFLAGS=-mod=vendor
-  - script: ./ci/test.sh
-    go: 1.13.x
-    env:
-    - GO111MODULE=on
-  - script: ./ci/test.sh
-    go: 1.14.x
-    env:
-    - GO111MODULE=on
-    - REPORT_COVERAGE=1
-    - BINDEPLOY=1
-
-before_deploy: /bin/bash ci/bintray.sh
-
-deploy:
-  on:
-    all_branches: true
-    condition: $BINDEPLOY = 1
-  provider: bintray
-  edge:
-    branch: v1.8.47
-  file: ci/deploy.json
-  user: 42wim
-  key:
-    secure: "CeXXe6JOmt7HYR81MdWLua0ltQHhDdkIeRGBFbgd7hkb1wi8eF9DgpAcQrTso8NIlHNZmSAP46uhFgsRvkuezzX0ygalZ7DCJyAyn3sAMEh+UQSHV1WGThRehTtidqRGjetzsIGSwdrJOWil+XTfbO1Z8DGzfakhSuAZka8CM4BAoe3YeP9rYK8h+84x0GHfczvsLtXZ3mWLvQuwe4pK6+ItBCUg0ae7O7ZUpWHy0xQQkkWztY/6RAzXfaG7DuGjIw+20fhx3WOXRNpHCtZ6Bc3qERCpk0s1HhlQWlrN9wDaFTBWYwlvSnNgvxxMbNXJ6RrRJ0l0bA7FUswYwyroxhzrGLdzWDg8dHaQkypocngdalfhpsnoO9j3ApJhomUFJ3UoEq5nOGRUrKn8MPi+dP0zE4kNQ3e4VNa1ufNrvfpWolMg3xh8OXuhQdD5wIM5zFAbRJLqWSCVAjPq4DDPecmvXBOlIial7oa312lN5qnBnUjvAcxszZ+FUyDHT1Grxzna4tMwxY9obPzZUzm7359AOCCwIQFVB8GLqD2nwIstcXS0zGRz+fhviPipHuBa02q5bGUZwmkvrSNab0s8Jo7pCrel2Rz3nWPKaiCfq2WjbW1CLheSMkOQrjsdUd1hhbqNWFPUjJPInTc77NAKCfm5runv5uyowRLh4NNd0sI="

Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:edge AS builder
+FROM alpine AS builder
 
 COPY . /go/src/github.com/42wim/matterbridge
 RUN apk update && apk add go git gcc musl-dev \
@@ -7,7 +7,10 @@ RUN apk update && apk add go git gcc musl-dev \
         && go get \
         && go build -x -ldflags "-X main.githash=$(git log --pretty=format:'%h' -n 1)" -o /bin/matterbridge
 
-FROM alpine:edge
+FROM alpine
 RUN apk --no-cache add ca-certificates mailcap
 COPY --from=builder /bin/matterbridge /bin/matterbridge
-ENTRYPOINT ["/bin/matterbridge"]
+RUN mkdir /etc/matterbridge \
+  && touch /etc/matterbridge/matterbridge.toml \
+  && ln -sf /matterbridge.toml /etc/matterbridge/matterbridge.toml
+ENTRYPOINT ["/bin/matterbridge", "-conf", "/etc/matterbridge/matterbridge.toml"]

README.md

@@ -9,27 +9,26 @@ Letting people be where they want to be.<br />
 
    <sup>
 
+[Discord][mb-discord] |
 [Gitter][mb-gitter] |
 [IRC][mb-irc] |
-[Discord][mb-discord] |
+[Keybase][mb-keybase] |
 [Matrix][mb-matrix] |
-[Slack][mb-slack] |
 [Mattermost][mb-mattermost] |
+[MSTeams][mb-msteams] |
 [Rocket.Chat][mb-rocketchat] |
-[XMPP][mb-xmpp] |
+[Slack][mb-slack] |
+[Telegram][mb-telegram] |
 [Twitch][mb-twitch] |
 [WhatsApp][mb-whatsapp] |
+[XMPP][mb-xmpp] |
 [Zulip][mb-zulip] |
-[Telegram][mb-telegram] |
-[Keybase][mb-keybase] |
-[MSTeams][mb-msteams] |
 And more...
 </sup>
 
 ---
 
 [![Download stable](https://img.shields.io/github/release/42wim/matterbridge.svg?label=download%20stable)](https://github.com/42wim/matterbridge/releases/latest)
-[![Download dev](https://img.shields.io/bintray/v/42wim/nightly/Matterbridge.svg?label=download%20dev&colorB=007ec6)](https://bintray.com/42wim/nightly/Matterbridge/_latestVersion)
 [![Maintainability](https://api.codeclimate.com/v1/badges/82dff70ef2ba85a6173a/maintainability)](https://codeclimate.com/github/42wim/matterbridge/maintainability)
 [![Test Coverage](https://api.codeclimate.com/v1/badges/82dff70ef2ba85a6173a/test_coverage)](https://codeclimate.com/github/42wim/matterbridge/test_coverage)<br />
 
@@ -87,30 +86,32 @@ And more...
 
 ### Natively supported
 
-- [Mattermost](https://github.com/mattermost/mattermost-server/) 4.x, 5.x
-- [IRC](http://www.mirc.com/servers.html)
-- [XMPP](https://xmpp.org)
-- [Gitter](https://gitter.im)
-- [Slack](https://slack.com)
 - [Discord](https://discordapp.com)
-- [Telegram](https://telegram.org)
-- [Rocket.chat](https://rocket.chat)
-- [Matrix](https://matrix.org)
-- [Microsoft Teams](https://teams.microsoft.com)
-- [Steam](https://store.steampowered.com/)
-- [Twitch](https://twitch.tv)
-- [Ssh-chat](https://github.com/shazow/ssh-chat)
-- [WhatsApp](https://www.whatsapp.com/)
-- [Zulip](https://zulipchat.com)
+- [Gitter](https://gitter.im)
+- [IRC](http://www.mirc.com/servers.html)
 - [Keybase](https://keybase.io)
+- [Matrix](https://matrix.org)
+- [Mattermost](https://github.com/mattermost/mattermost-server/) 4.x, 5.x
+- [Microsoft Teams](https://teams.microsoft.com)
+- [Nextcloud Talk](https://nextcloud.com/talk/)
+- [Rocket.chat](https://rocket.chat)
+- [Slack](https://slack.com)
+- [Ssh-chat](https://github.com/shazow/ssh-chat)
+- [Steam](https://store.steampowered.com/)
+- [Telegram](https://telegram.org)
+- [Twitch](https://twitch.tv)
+- [WhatsApp](https://www.whatsapp.com/)
+- [XMPP](https://xmpp.org)
+- [Zulip](https://zulipchat.com)
 
 ### 3rd party via matterbridge api
 
+- [Discourse](https://github.com/DeclanHoare/matterbabble)
+- [Facebook messenger](https://github.com/VictorNine/fbridge)
 - [Minecraft](https://github.com/elytra/MatterLink)
 - [Reddit](https://github.com/bonehurtingjuice/mattereddit)
-- [Facebook messenger](https://github.com/VictorNine/fbridge)
-- [Discourse](https://github.com/DeclanHoare/matterbabble)
 - [Counter-Strike, half-life and more](https://forums.alliedmods.net/showthread.php?t=319430)
+- [MatterAMXX](https://github.com/GabeIggy/MatterAMXX)
 
 ### API
 
@@ -130,35 +131,36 @@ Used by the projects below. Feel free to make a PR to add your project to this l
 
 Questions or want to test on your favorite platform? Join below:
 
+- [Discord][mb-discord]
 - [Gitter][mb-gitter]
 - [IRC][mb-irc]
-- [Discord][mb-discord]
+- [Keybase][mb-keybase]
 - [Matrix][mb-matrix]
-- [Slack][mb-slack]
 - [Mattermost][mb-mattermost]
 - [Rocket.Chat][mb-rocketchat]
-- [XMPP][mb-xmpp] (matterbridge@conference.jabber.de)
-- [Twitch][mb-twitch]
-- [Zulip][mb-zulip]
+- [Slack][mb-slack]
 - [Telegram][mb-telegram]
-- [Keybase][mb-keybase]
+- [Twitch][mb-twitch]
+- [XMPP][mb-xmpp] (matterbridge@conference.jabber.de)
+- [Zulip][mb-zulip]
 
 ## Screenshots
 
-See https://github.com/42wim/matterbridge/wiki
+See <https://github.com/42wim/matterbridge/wiki>
 
 ## Installing / upgrading
 
 ### Binaries
 
-- Latest stable release [v1.17.1](https://github.com/42wim/matterbridge/releases/latest)
-- Development releases (follows master) can be downloaded [here](https://dl.bintray.com/42wim/nightly/)
+- Latest stable release [v1.18.3](https://github.com/42wim/matterbridge/releases/latest)
+- Development releases (follows master) can be downloaded [here](https://github.com/42wim/matterbridge/actions) selecting the latest green build and then artifacts.
 
 To install or upgrade just download the latest [binary](https://github.com/42wim/matterbridge/releases/latest) and follow the instructions on the [howto](https://github.com/42wim/matterbridge/wiki/How-to-create-your-config) for a step by step walkthrough for creating your configuration.
 
 ### Packages
 
 - [Overview](https://repology.org/metapackage/matterbridge/versions)
+- [snap](https://snapcraft.io/matterbridge)
 
 ## Building
 
@@ -167,14 +169,13 @@ Most people just want to use binaries, you can find those [here](https://github.
 If you really want to build from source, follow these instructions:
 Go 1.12+ is required. Make sure you have [Go](https://golang.org/doc/install) properly installed.
 
-
-```
+```bash
 go get github.com/42wim/matterbridge
 ```
 
 You should now have matterbridge binary in the ~/go/bin directory:
 
-```
+```bash
 $ ls ~/go/bin/
 matterbridge
 ```
@@ -257,7 +258,7 @@ RemoteNickFormat="[{PROTOCOL}/{BRIDGE}] <{NICK}> "
 
 See [howto](https://github.com/42wim/matterbridge/wiki/How-to-create-your-config) for a step by step walkthrough for creating your configuration.
 
-```
+```bash
 Usage of ./matterbridge:
   -conf string
         config file (default "matterbridge.toml")
@@ -298,14 +299,14 @@ See [FAQ](https://github.com/42wim/matterbridge/wiki/FAQ)
 ## Articles
 
 - [matterbridge on kubernetes](https://medium.freecodecamp.org/using-kubernetes-to-deploy-a-chat-gateway-or-when-technology-works-like-its-supposed-to-a169a8cd69a3)
-- https://mattermost.com/blog/connect-irc-to-mattermost/
-- https://blog.valvin.fr/2016/09/17/mattermost-et-un-channel-irc-cest-possible/
-- https://blog.brightscout.com/top-10-mattermost-integrations/
-- http://bencey.co.nz/2018/09/17/bridge/
-- https://www.algoo.fr/blog/2018/01/19/recouvrez-votre-liberte-en-quittant-slack-pour-un-mattermost-auto-heberge/
-- https://kopano.com/blog/matterbridge-bridging-mattermost-chat/
-- https://www.stitcher.com/s/?eid=52382713
-- https://daniele.tech/2019/02/how-to-use-matterbridge-to-connect-2-different-slack-workspaces/
+- <https://mattermost.com/blog/connect-irc-to-mattermost/>
+- <https://blog.valvin.fr/2016/09/17/mattermost-et-un-channel-irc-cest-possible/>
+- <https://blog.brightscout.com/top-10-mattermost-integrations/>
+- <https://www.algoo.fr/blog/2018/01/19/recouvrez-votre-liberte-en-quittant-slack-pour-un-mattermost-auto-heberge/>
+- <https://kopano.com/blog/matterbridge-bridging-mattermost-chat/>
+- <https://www.stitcher.com/s/?eid=52382713>
+- <https://daniele.tech/2019/02/how-to-use-matterbridge-to-connect-2-different-slack-workspaces/>
+- <https://userlinux.net/mattermost-and-matterbridge.html>
 
 ## Thanks
 
@@ -318,38 +319,39 @@ See [FAQ](https://github.com/42wim/matterbridge/wiki/FAQ)
 
 Matterbridge wouldn't exist without these libraries:
 
-- discord - https://github.com/bwmarrin/discordgo
-- echo - https://github.com/labstack/echo
-- gitter - https://github.com/sromku/go-gitter
-- gops - https://github.com/google/gops
-- gozulipbot - https://github.com/ifo/gozulipbot
-- irc - https://github.com/lrstanley/girc
-- mattermost - https://github.com/mattermost/mattermost-server
-- matrix - https://github.com/matrix-org/gomatrix
-- sshchat - https://github.com/shazow/ssh-chat
-- slack - https://github.com/nlopes/slack
-- steam - https://github.com/Philipp15b/go-steam
-- telegram - https://github.com/go-telegram-bot-api/telegram-bot-api
-- xmpp - https://github.com/mattn/go-xmpp
-- whatsapp - https://github.com/Rhymen/go-whatsapp/
-- zulip - https://github.com/ifo/gozulipbot
-- tengo - https://github.com/d5/tengo
-- keybase - https://github.com/keybase/go-keybase-chat-bot
-- msgraph.go - https://github.com/yaegashi/msgraph.go
+- discord - <https://github.com/bwmarrin/discordgo>
+- echo - <https://github.com/labstack/echo>
+- gitter - <https://github.com/sromku/go-gitter>
+- gops - <https://github.com/google/gops>
+- gozulipbot - <https://github.com/ifo/gozulipbot>
+- irc - <https://github.com/lrstanley/girc>
+- keybase - <https://github.com/keybase/go-keybase-chat-bot>
+- matrix - <https://github.com/matrix-org/gomatrix>
+- mattermost - <https://github.com/mattermost/mattermost-server>
+- msgraph.go - <https://github.com/yaegashi/msgraph.go>
+- nctalk - <https://github.com/gary-kim/go-nc-talk>
+- slack - <https://github.com/nlopes/slack>
+- sshchat - <https://github.com/shazow/ssh-chat>
+- steam - <https://github.com/Philipp15b/go-steam>
+- telegram - <https://github.com/go-telegram-bot-api/telegram-bot-api>
+- tengo - <https://github.com/d5/tengo>
+- whatsapp - <https://github.com/Rhymen/go-whatsapp>
+- xmpp - <https://github.com/mattn/go-xmpp>
+- zulip - <https://github.com/ifo/gozulipbot>
 
 <!-- Links -->
 
+[mb-discord]: https://discord.gg/AkKPtrQ
 [mb-gitter]: https://gitter.im/42wim/matterbridge
 [mb-irc]: https://webchat.freenode.net/?channels=matterbridgechat
-[mb-discord]: https://discord.gg/AkKPtrQ
+[mb-keybase]: https://keybase.io/team/matterbridge
 [mb-matrix]: https://riot.im/app/#/room/#matterbridge:matrix.org
-[mb-slack]: https://join.slack.com/matterbridgechat/shared_invite/MjEwODMxNjU1NDMwLTE0OTk2MTU3NTMtMzZkZmRiNDZhOA
 [mb-mattermost]: https://framateam.org/signup_user_complete/?id=tfqm33ggop8x3qgu4boeieta6e
+[mb-msteams]: https://teams.microsoft.com/join/hj92x75gd3y7
 [mb-rocketchat]: https://open.rocket.chat/channel/matterbridge
-[mb-xmpp]: https://inverse.chat/
+[mb-slack]: https://join.slack.com/matterbridgechat/shared_invite/MjEwODMxNjU1NDMwLTE0OTk2MTU3NTMtMzZkZmRiNDZhOA
+[mb-telegram]: https://t.me/Matterbridge
 [mb-twitch]: https://www.twitch.tv/matterbridge
 [mb-whatsapp]: https://www.whatsapp.com/
-[mb-keybase]: https://keybase.io/team/matterbridge
+[mb-xmpp]: https://inverse.chat/
 [mb-zulip]: https://matterbridge.zulipchat.com/register/
-[mb-telegram]: https://t.me/Matterbridge
-[mb-msteams]: https://teams.microsoft.com/join/hj92x75gd3y7

bridge/api/api.go

@@ -6,17 +6,20 @@ import (
 	"sync"
 	"time"
 
+	"gopkg.in/olahol/melody.v1"
+
 	"github.com/42wim/matterbridge/bridge"
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/labstack/echo/v4"
 	"github.com/labstack/echo/v4/middleware"
-	"github.com/zfjagann/golang-ring"
+	ring "github.com/zfjagann/golang-ring"
 )
 
 type API struct {
 	Messages ring.Ring
 	sync.RWMutex
 	*bridge.Config
+	mrouter *melody.Melody
 }
 
 type Message struct {
@@ -32,6 +35,32 @@ func New(cfg *bridge.Config) bridge.Bridger {
 	e := echo.New()
 	e.HideBanner = true
 	e.HidePort = true
+
+	b.mrouter = melody.New()
+	b.mrouter.HandleMessage(func(s *melody.Session, msg []byte) {
+		message := config.Message{}
+		err := json.Unmarshal(msg, &message)
+		if err != nil {
+			b.Log.Errorf("failed to decode message from byte[] '%s'", string(msg))
+			return
+		}
+		b.handleWebsocketMessage(message, s)
+	})
+	b.mrouter.HandleConnect(func(session *melody.Session) {
+		greet := b.getGreeting()
+		data, err := json.Marshal(greet)
+		if err != nil {
+			b.Log.Errorf("failed to encode message '%v'", greet)
+			return
+		}
+		err = session.Write(data)
+		if err != nil {
+			b.Log.Errorf("failed to write message '%s'", string(data))
+			return
+		}
+		// TODO: send message history buffer from `b.Messages` here
+	})
+
 	b.Messages = ring.Ring{}
 	if b.GetInt("Buffer") != 0 {
 		b.Messages.SetCapacity(b.GetInt("Buffer"))
@@ -41,9 +70,17 @@ func New(cfg *bridge.Config) bridge.Bridger {
 			return key == b.GetString("Token"), nil
 		}))
 	}
+
+	// Set RemoteNickFormat to a sane default
+	if !b.IsKeySet("RemoteNickFormat") {
+		b.Log.Debugln("RemoteNickFormat is unset, defaulting to \"{NICK}\"")
+		b.Config.Config.Viper().Set(b.GetConfigKey("RemoteNickFormat"), "{NICK}")
+	}
+
 	e.GET("/api/health", b.handleHealthcheck)
 	e.GET("/api/messages", b.handleMessages)
 	e.GET("/api/stream", b.handleStream)
+	e.GET("/api/websocket", b.handleWebsocket)
 	e.POST("/api/message", b.handlePostMessage)
 	go func() {
 		if b.GetString("BindAddress") == "" {
@@ -58,13 +95,13 @@ func New(cfg *bridge.Config) bridge.Bridger {
 func (b *API) Connect() error {
 	return nil
 }
+
 func (b *API) Disconnect() error {
 	return nil
-
 }
+
 func (b *API) JoinChannel(channel config.ChannelInfo) error {
 	return nil
-
 }
 
 func (b *API) Send(msg config.Message) (string, error) {
@@ -74,7 +111,14 @@ func (b *API) Send(msg config.Message) (string, error) {
 	if msg.Event == config.EventMsgDelete {
 		return "", nil
 	}
-	b.Messages.Enqueue(&msg)
+	b.Log.Debugf("enqueueing message from %s on ring buffer", msg.Username)
+	b.Messages.Enqueue(msg)
+
+	data, err := json.Marshal(msg)
+	if err != nil {
+		b.Log.Errorf("failed to encode message  '%s'", msg)
+	}
+	_ = b.mrouter.Broadcast(data)
 	return "", nil
 }
 
@@ -106,18 +150,23 @@ func (b *API) handleMessages(c echo.Context) error {
 	return nil
 }
 
-func (b *API) handleStream(c echo.Context) error {
-	c.Response().Header().Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
-	c.Response().WriteHeader(http.StatusOK)
-	greet := config.Message{
+func (b *API) getGreeting() config.Message {
+	return config.Message{
 		Event:     config.EventAPIConnected,
 		Timestamp: time.Now(),
 	}
+}
+
+func (b *API) handleStream(c echo.Context) error {
+	c.Response().Header().Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
+	c.Response().WriteHeader(http.StatusOK)
+	greet := b.getGreeting()
 	if err := json.NewEncoder(c.Response()).Encode(greet); err != nil {
 		return err
 	}
 	c.Response().Flush()
 	for {
+		// TODO: this causes issues, messages should be broadcasted to all connected clients
 		msg := b.Messages.Dequeue()
 		if msg != nil {
 			if err := json.NewEncoder(c.Response()).Encode(msg); err != nil {
@@ -128,3 +177,31 @@ func (b *API) handleStream(c echo.Context) error {
 		time.Sleep(200 * time.Millisecond)
 	}
 }
+
+func (b *API) handleWebsocketMessage(message config.Message, s *melody.Session) {
+	message.Channel = "api"
+	message.Protocol = "api"
+	message.Account = b.Account
+	message.ID = ""
+	message.Timestamp = time.Now()
+
+	data, err := json.Marshal(message)
+	if err != nil {
+		b.Log.Errorf("failed to encode message for loopback '%v'", message)
+		return
+	}
+	_ = b.mrouter.BroadcastOthers(data, s)
+
+	b.Log.Debugf("Sending websocket message from %s on %s to gateway", message.Username, "api")
+	b.Remote <- message
+}
+
+func (b *API) handleWebsocket(c echo.Context) error {
+	err := b.mrouter.HandleRequest(c.Response(), c.Request())
+	if err != nil {
+		b.Log.Errorf("error in websocket handling  '%v'", err)
+		return err
+	}
+
+	return nil
+}

bridge/bridge.go

@@ -4,6 +4,7 @@ import (
 	"log"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/sirupsen/logrus"
@@ -74,6 +75,7 @@ func (b *Bridge) joinChannels(channels map[string]config.ChannelInfo, exists map
 	for ID, channel := range channels {
 		if !exists[ID] {
 			b.Log.Infof("%s: joining %s (ID: %s)", b.Account, channel.Name, ID)
+			time.Sleep(time.Duration(b.GetInt("JoinDelay")) * time.Millisecond)
 			err := b.JoinChannel(channel)
 			if err != nil {
 				return err
@@ -84,8 +86,16 @@ func (b *Bridge) joinChannels(channels map[string]config.ChannelInfo, exists map
 	return nil
 }
 
+func (b *Bridge) GetConfigKey(key string) string {
+	return b.Account + "." + key
+}
+
+func (b *Bridge) IsKeySet(key string) bool {
+	return b.Config.IsKeySet(b.GetConfigKey(key)) || b.Config.IsKeySet("general."+key)
+}
+
 func (b *Bridge) GetBool(key string) bool {
-	val, ok := b.Config.GetBool(b.Account + "." + key)
+	val, ok := b.Config.GetBool(b.GetConfigKey(key))
 	if !ok {
 		val, _ = b.Config.GetBool("general." + key)
 	}
@@ -93,7 +103,7 @@ func (b *Bridge) GetBool(key string) bool {
 }
 
 func (b *Bridge) GetInt(key string) int {
-	val, ok := b.Config.GetInt(b.Account + "." + key)
+	val, ok := b.Config.GetInt(b.GetConfigKey(key))
 	if !ok {
 		val, _ = b.Config.GetInt("general." + key)
 	}
@@ -101,7 +111,7 @@ func (b *Bridge) GetInt(key string) int {
 }
 
 func (b *Bridge) GetString(key string) string {
-	val, ok := b.Config.GetString(b.Account + "." + key)
+	val, ok := b.Config.GetString(b.GetConfigKey(key))
 	if !ok {
 		val, _ = b.Config.GetString("general." + key)
 	}
@@ -109,7 +119,7 @@ func (b *Bridge) GetString(key string) string {
 }
 
 func (b *Bridge) GetStringSlice(key string) []string {
-	val, ok := b.Config.GetStringSlice(b.Account + "." + key)
+	val, ok := b.Config.GetStringSlice(b.GetConfigKey(key))
 	if !ok {
 		val, _ = b.Config.GetStringSlice("general." + key)
 	}
@@ -117,7 +127,7 @@ func (b *Bridge) GetStringSlice(key string) []string {
 }
 
 func (b *Bridge) GetStringSlice2D(key string) [][]string {
-	val, ok := b.Config.GetStringSlice2D(b.Account + "." + key)
+	val, ok := b.Config.GetStringSlice2D(b.GetConfigKey(key))
 	if !ok {
 		val, _ = b.Config.GetStringSlice2D("general." + key)
 	}

bridge/config/config.go

@@ -3,6 +3,7 @@ package config
 import (
 	"bytes"
 	"io/ioutil"
+	"os"
 	"path/filepath"
 	"strings"
 	"sync"
@@ -84,18 +85,22 @@ type Protocol struct {
 	DisableWebPagePreview  bool   // telegram
 	EditSuffix             string // mattermost, slack, discord, telegram, gitter
 	EditDisable            bool   // mattermost, slack, discord, telegram, gitter
+	HTMLDisable            bool   // matrix
 	IconURL                string // mattermost, slack
 	IgnoreFailureOnStart   bool   // general
 	IgnoreNicks            string // all protocols
 	IgnoreMessages         string // all protocols
 	Jid                    string // xmpp
+	JoinDelay              string // all protocols
 	Label                  string // all protocols
 	Login                  string // mattermost, matrix
+	LogFile                string // general
 	MediaDownloadBlackList []string
 	MediaDownloadPath      string // Basically MediaServerUpload, but instead of uploading it, just write it to a file on the same server.
 	MediaDownloadSize      int    // all protocols
 	MediaServerDownload    string
 	MediaServerUpload      string
+	MediaConvertTgs        string     // telegram
 	MediaConvertWebPToPNG  bool       // telegram
 	MessageDelay           int        // IRC, time in millisecond to wait between messages
 	MessageFormat          string     // telegram
@@ -134,6 +139,7 @@ type Protocol struct {
 	SkipTLSVerify          bool       // IRC, mattermost
 	SkipVersionCheck       bool       // mattermost
 	StripNick              bool       // all protocols
+	StripMarkdown          bool       // irc
 	SyncTopic              bool       // slack
 	TengoModifyMessage     string     // general
 	Team                   string     // mattermost, keybase
@@ -216,6 +222,7 @@ type BridgeValues struct {
 type Config interface {
 	Viper() *viper.Viper
 	BridgeValues() *BridgeValues
+	IsKeySet(key string) bool
 	GetBool(key string) (bool, bool)
 	GetInt(key string) (int, bool)
 	GetString(key string) (string, bool)
@@ -243,6 +250,15 @@ func NewConfig(rootLogger *logrus.Logger, cfgfile string) Config {
 
 	cfgtype := detectConfigType(cfgfile)
 	mycfg := newConfigFromString(logger, input, cfgtype)
+	if mycfg.cv.General.LogFile != "" {
+		logfile, err := os.OpenFile(mycfg.cv.General.LogFile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600)
+		if err == nil {
+			logger.Info("Opening log file ", mycfg.cv.General.LogFile)
+			rootLogger.Out = logfile
+		} else {
+			logger.Warn("Failed to open ", mycfg.cv.General.LogFile)
+		}
+	}
 	if mycfg.cv.General.MediaDownloadSize == 0 {
 		mycfg.cv.General.MediaDownloadSize = 1000000
 	}
@@ -300,6 +316,12 @@ func (c *config) Viper() *viper.Viper {
 	return c.v
 }
 
+func (c *config) IsKeySet(key string) bool {
+	c.RLock()
+	defer c.RUnlock()
+	return c.v.IsSet(key)
+}
+
 func (c *config) GetBool(key string) (bool, bool) {
 	c.RLock()
 	defer c.RUnlock()
@@ -359,6 +381,11 @@ type TestConfig struct {
 	Overrides map[string]interface{}
 }
 
+func (c *TestConfig) IsKeySet(key string) bool {
+	_, ok := c.Overrides[key]
+	return ok || c.Config.IsKeySet(key)
+}
+
 func (c *TestConfig) GetBool(key string) (bool, bool) {
 	val, ok := c.Overrides[key]
 	if ok {

bridge/discord/discord.go

@@ -34,6 +34,8 @@ type Bdiscord struct {
 	membersMutex  sync.RWMutex
 	userMemberMap map[string]*discordgo.Member
 	nickMemberMap map[string]*discordgo.Member
+	webhookCache  map[string]string
+	webhookMutex  sync.RWMutex
 }
 
 func New(cfg *bridge.Config) bridge.Bridger {
@@ -41,6 +43,7 @@ func New(cfg *bridge.Config) bridge.Bridger {
 	b.userMemberMap = make(map[string]*discordgo.Member)
 	b.nickMemberMap = make(map[string]*discordgo.Member)
 	b.channelInfoMap = make(map[string]*config.ChannelInfo)
+	b.webhookCache = make(map[string]string)
 	if b.GetString("WebhookURL") != "" {
 		b.Log.Debug("Configuring Discord Incoming Webhook")
 		b.webhookID, b.webhookToken = b.splitURL(b.GetString("WebhookURL"))
@@ -188,6 +191,8 @@ func (b *Bdiscord) JoinChannel(channel config.ChannelInfo) error {
 func (b *Bdiscord) Send(msg config.Message) (string, error) {
 	b.Log.Debugf("=> Receiving %#v", msg)
 
+	origMsgID := msg.ID
+
 	channelID := b.getChannelID(msg.Channel)
 	if channelID == "" {
 		return "", fmt.Errorf("Could not find channelID for %v", msg.Channel)
@@ -224,12 +229,13 @@ func (b *Bdiscord) Send(msg config.Message) (string, error) {
 	// Use webhook to send the message
 	if wID != "" && msg.Event != config.EventMsgDelete {
 		// skip events
-		if msg.Event != "" && msg.Event != config.EventJoinLeave && msg.Event != config.EventTopicChange {
+		if msg.Event != "" && msg.Event != config.EventUserAction && msg.Event != config.EventJoinLeave && msg.Event != config.EventTopicChange {
 			return "", nil
 		}
 
 		// If we are editing a message, delete the old message
 		if msg.ID != "" {
+			msg.ID = b.getCacheID(msg.ID)
 			b.Log.Debugf("Deleting edited webhook message")
 			err := b.c.ChannelMessageDelete(channelID, msg.ID)
 			if err != nil {
@@ -273,6 +279,8 @@ func (b *Bdiscord) Send(msg config.Message) (string, error) {
 		if msg == nil {
 			return "", nil
 		}
+
+		b.updateCacheID(origMsgID, msg.ID)
 		return msg.ID, nil
 	}
 
@@ -283,6 +291,7 @@ func (b *Bdiscord) Send(msg config.Message) (string, error) {
 		if msg.ID == "" {
 			return "", nil
 		}
+		msg.ID = b.getCacheID(msg.ID)
 		err := b.c.ChannelMessageDelete(channelID, msg.ID)
 		return "", err
 	}

bridge/discord/handlers.go

@@ -217,7 +217,7 @@ func handleEmbed(embed *discordgo.MessageEmbed) string {
 
 		i++
 		if i == 1 {
-			result += "embed: " + e
+			result += " embed: " + e
 			continue
 		}
 

bridge/discord/handlers_test.go

@@ -20,14 +20,14 @@ func TestHandleEmbed(t *testing.T) {
 			embed: &discordgo.MessageEmbed{
 				Title: "blah",
 			},
-			result: "embed: blah\n",
+			result: " embed: blah\n",
 		},
 		"two": {
 			embed: &discordgo.MessageEmbed{
 				Title:       "blah",
 				Description: "blah2",
 			},
-			result: "embed: blah - blah2\n",
+			result: " embed: blah - blah2\n",
 		},
 		"three": {
 			embed: &discordgo.MessageEmbed{
@@ -35,20 +35,20 @@ func TestHandleEmbed(t *testing.T) {
 				Description: "blah2",
 				URL:         "blah3",
 			},
-			result: "embed: blah - blah2 - blah3\n",
+			result: " embed: blah - blah2 - blah3\n",
 		},
 		"twob": {
 			embed: &discordgo.MessageEmbed{
 				Description: "blah2",
 				URL:         "blah3",
 			},
-			result: "embed: blah2 - blah3\n",
+			result: " embed: blah2 - blah3\n",
 		},
 		"oneb": {
 			embed: &discordgo.MessageEmbed{
 				URL: "blah3",
 			},
-			result: "embed: blah3\n",
+			result: " embed: blah3\n",
 		},
 	}
 

bridge/discord/helpers.go

@@ -188,8 +188,9 @@ func replaceEmotes(text string) string {
 }
 
 func (b *Bdiscord) replaceAction(text string) (string, bool) {
-	if strings.HasPrefix(text, "_") && strings.HasSuffix(text, "_") {
-		return text[1 : len(text)-1], true
+	length := len(text)
+	if length > 1 && text[0] == '_' && text[length-1] == '_' {
+		return text[1 : length-1], true
 	}
 	return text, false
 }
@@ -208,6 +209,40 @@ func (b *Bdiscord) splitURL(url string) (string, string) {
 	return webhookURLSplit[webhookIdxID], webhookURLSplit[webhookIdxToken]
 }
 
+// getcacheID tries to find a corresponding msgID in the webhook cache.
+// if not found returns the original request.
+func (b *Bdiscord) getCacheID(msgID string) string {
+	b.webhookMutex.RLock()
+	defer b.webhookMutex.RUnlock()
+	for k, v := range b.webhookCache {
+		if msgID == k {
+			return v
+		}
+	}
+	return msgID
+}
+
+// updateCacheID updates the cache so that the newID takes the place of
+// the original ID. This is used for edit/deletes in combination with webhooks
+// as editing a message via webhook means deleting the message and creating a
+// new message (with a new ID). This ID needs to be set instead of the original ID
+func (b *Bdiscord) updateCacheID(origID, newID string) {
+	b.webhookMutex.Lock()
+	match := false
+	for k, v := range b.webhookCache {
+		if v == origID {
+			delete(b.webhookCache, k)
+			b.webhookCache[origID] = newID
+			match = true
+			continue
+		}
+	}
+	if !match && origID != "" {
+		b.webhookCache[origID] = newID
+	}
+	b.webhookMutex.Unlock()
+}
+
 func enumerateUsernames(s string) []string {
 	onlySpace := true
 	for _, r := range s {

bridge/helper/helper.go

@@ -5,7 +5,10 @@ import (
 	"fmt"
 	"image/png"
 	"io"
+	"io/ioutil"
 	"net/http"
+	"os"
+	"os/exec"
 	"regexp"
 	"strings"
 	"time"
@@ -192,7 +195,7 @@ func ParseMarkdown(input string) string {
 	return res
 }
 
-// ConvertWebPToPNG convert input data (which should be WebP format to PNG format)
+// ConvertWebPToPNG converts input data (which should be WebP format) to PNG format
 func ConvertWebPToPNG(data *[]byte) error {
 	r := bytes.NewReader(*data)
 	m, err := webp.Decode(r)
@@ -207,3 +210,49 @@ func ConvertWebPToPNG(data *[]byte) error {
 	*data = w.Bytes()
 	return nil
 }
+
+// CanConvertTgsToX Checks whether the external command necessary for ConvertTgsToX works.
+func CanConvertTgsToX() error {
+	// We depend on the fact that `lottie_convert.py --help` has exit status 0.
+	// Hyrum's Law predicted this, and Murphy's Law predicts that this will break eventually.
+	// However, there is no alternative like `lottie_convert.py --is-properly-installed`
+	cmd := exec.Command("lottie_convert.py", "--help")
+	return cmd.Run()
+}
+
+// ConvertTgsToWebP convert input data (which should be tgs format) to WebP format
+// This relies on an external command, which is ugly, but works.
+func ConvertTgsToX(data *[]byte, outputFormat string, logger *logrus.Entry) error {
+	// lottie can't handle input from a pipe, so write to a temporary file:
+	tmpFile, err := ioutil.TempFile(os.TempDir(), "matterbridge-lottie-*.tgs")
+	if err != nil {
+		return err
+	}
+	tmpFileName := tmpFile.Name()
+	defer func() {
+		if removeErr := os.Remove(tmpFileName); removeErr != nil {
+			logger.Errorf("Could not delete temporary file %s: %v", tmpFileName, removeErr)
+		}
+	}()
+
+	if _, writeErr := tmpFile.Write(*data); writeErr != nil {
+		return writeErr
+	}
+	// Must close before calling lottie to avoid data races:
+	if closeErr := tmpFile.Close(); closeErr != nil {
+		return closeErr
+	}
+
+	// Call lottie to transform:
+	cmd := exec.Command("lottie_convert.py", "--input-format", "lottie", "--output-format", outputFormat, tmpFileName, "/dev/stdout")
+	cmd.Stderr = nil
+	// NB: lottie writes progress into to stderr in all cases.
+	stdout, stderr := cmd.Output()
+	if stderr != nil {
+		// 'stderr' already contains some parts of Stderr, because it was set to 'nil'.
+		return stderr
+	}
+
+	*data = stdout
+	return nil
+}

bridge/irc/handlers.go

@@ -10,8 +10,8 @@ import (
 
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/42wim/matterbridge/bridge/helper"
-	"github.com/dfordsoft/golib/ic"
 	"github.com/lrstanley/girc"
+	"github.com/missdeer/golib/ic"
 	"github.com/paulrosania/go-charset/charset"
 	"github.com/saintfish/chardet"
 
@@ -54,12 +54,12 @@ func (b *Birc) handleFiles(msg *config.Message) bool {
 	for _, f := range msg.Extra["file"] {
 		fi := f.(config.FileInfo)
 		if fi.Comment != "" {
-			msg.Text += fi.Comment + ": "
+			msg.Text += fi.Comment + " : "
 		}
 		if fi.URL != "" {
 			msg.Text = fi.URL
 			if fi.Comment != "" {
-				msg.Text = fi.Comment + ": " + fi.URL
+				msg.Text = fi.Comment + " : " + fi.URL
 			}
 		}
 		b.Local <- config.Message{Text: msg.Text, Username: msg.Username, Channel: msg.Channel, Event: msg.Event}

bridge/irc/irc.go

@@ -14,6 +14,7 @@ import (
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/42wim/matterbridge/bridge/helper"
 	"github.com/lrstanley/girc"
+	stripmd "github.com/writeas/go-strip-markdown"
 
 	// We need to import the 'data' package as an implicit dependency.
 	// See: https://godoc.org/github.com/paulrosania/go-charset/charset
@@ -156,6 +157,10 @@ func (b *Birc) Send(msg config.Message) (string, error) {
 	}
 
 	var msgLines []string
+	if b.GetBool("StripMarkdown") {
+		msg.Text = stripmd.Strip(msg.Text)
+	}
+
 	if b.GetBool("MessageSplit") {
 		msgLines = helper.GetSubLines(msg.Text, b.MessageLength)
 	} else {
@@ -201,7 +206,7 @@ func (b *Birc) doSend() {
 	for msg := range b.Local {
 		<-throttle.C
 		username := msg.Username
-		if b.GetBool("Colornicks") {
+		if b.GetBool("Colornicks") && len(username) > 1 {
 			checksum := crc32.ChecksumIEEE([]byte(msg.Username))
 			colorCode := checksum%14 + 2 // quick fix - prevent white or black color codes
 			username = fmt.Sprintf("\x03%02d%s\x0F", colorCode, msg.Username)
@@ -245,6 +250,8 @@ func (b *Birc) getClient() (*girc.Client, error) {
 		SSL:        b.GetBool("UseTLS"),
 		TLSConfig:  &tls.Config{InsecureSkipVerify: b.GetBool("SkipTLSVerify"), ServerName: server}, //nolint:gosec
 		PingDelay:  time.Minute,
+		// skip gIRC internal rate limiting, since we have our own throttling
+		AllowFlood: true,
 	})
 	return i, nil
 }

bridge/matrix/matrix.go

@@ -2,17 +2,19 @@ package bmatrix
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"html"
 	"mime"
 	"regexp"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/42wim/matterbridge/bridge"
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/42wim/matterbridge/bridge/helper"
-	matrix "github.com/matterbridge/gomatrix"
+	matrix "github.com/matrix-org/gomatrix"
 )
 
 type Bmatrix struct {
@@ -20,13 +22,21 @@ type Bmatrix struct {
 	UserID  string
 	RoomMap map[string]string
 	sync.RWMutex
-	htmlTag *regexp.Regexp
+	htmlTag            *regexp.Regexp
+	htmlReplacementTag *regexp.Regexp
 	*bridge.Config
 }
 
+type httpError struct {
+	Errcode      string `json:"errcode"`
+	Err          string `json:"error"`
+	RetryAfterMs int    `json:"retry_after_ms"`
+}
+
 func New(cfg *bridge.Config) bridge.Bridger {
 	b := &Bmatrix{Config: cfg}
 	b.htmlTag = regexp.MustCompile("</.*?>")
+	b.htmlReplacementTag = regexp.MustCompile("<[^>]*>")
 	b.RoomMap = make(map[string]string)
 	return b
 }
@@ -39,9 +49,10 @@ func (b *Bmatrix) Connect() error {
 		return err
 	}
 	resp, err := b.mc.Login(&matrix.ReqLogin{
-		Type:     "m.login.password",
-		User:     b.GetString("Login"),
-		Password: b.GetString("Password"),
+		Type:       "m.login.password",
+		User:       b.GetString("Login"),
+		Password:   b.GetString("Password"),
+		Identifier: matrix.NewUserIdentifier(b.GetString("Login")),
 	})
 	if err != nil {
 		return err
@@ -58,14 +69,25 @@ func (b *Bmatrix) Disconnect() error {
 }
 
 func (b *Bmatrix) JoinChannel(channel config.ChannelInfo) error {
+retry:
 	resp, err := b.mc.JoinRoom(channel.Name, "", nil)
 	if err != nil {
+		httpErr := handleError(err)
+		if httpErr.Errcode == "M_LIMIT_EXCEEDED" {
+			b.Log.Infof("getting ratelimited by matrix, sleeping approx %d seconds before joining %s", httpErr.RetryAfterMs/1000, channel.Name)
+			time.Sleep((time.Duration(httpErr.RetryAfterMs) * time.Millisecond))
+
+			goto retry
+		}
+
 		return err
 	}
+
 	b.Lock()
 	b.RoomMap[resp.RoomID] = channel.Name
 	b.Unlock()
-	return err
+
+	return nil
 }
 
 func (b *Bmatrix) Send(msg config.Message) (string, error) {
@@ -124,13 +146,28 @@ func (b *Bmatrix) Send(msg config.Message) (string, error) {
 		return resp.EventID, err
 	}
 
-	username := html.EscapeString(msg.Username)
+	if b.GetBool("HTMLDisable") {
+		resp, err := b.mc.SendText(channel, msg.Username+msg.Text)
+		if err != nil {
+			return "", err
+		}
+		return resp.EventID, err
+	}
+
+	var username string
+	var plainUsername string
 	// check if we have a </tag>. if we have, we don't escape HTML. #696
 	if b.htmlTag.MatchString(msg.Username) {
 		username = msg.Username
+		// remove the HTML formatting for beautiful push messages #1188
+		plainUsername = b.htmlReplacementTag.ReplaceAllString(msg.Username, "")
+	} else {
+		username = html.EscapeString(msg.Username)
+		plainUsername = msg.Username
 	}
+
 	// Post normal message with HTML support (eg riot.im)
-	resp, err := b.mc.SendHTML(channel, msg.Username+msg.Text, username+helper.ParseMarkdown(msg.Text))
+	resp, err := b.mc.SendFormattedText(channel, plainUsername+msg.Text, username+helper.ParseMarkdown(msg.Text))
 	if err != nil {
 		return "", err
 	}
@@ -334,13 +371,29 @@ func (b *Bmatrix) handleUploadFile(msg *config.Message, channel string, fi *conf
 		}
 	case strings.Contains(mtype, "application"):
 		b.Log.Debugf("sendFile %s", res.ContentURI)
-		_, err = b.mc.SendFile(channel, fi.Name, res.ContentURI, mtype, uint(len(*fi.Data)))
+		_, err = b.mc.SendMessageEvent(channel, "m.room.message", matrix.FileMessage{
+			MsgType: "m.file",
+			Body:    fi.Name,
+			URL:     res.ContentURI,
+			Info: matrix.FileInfo{
+				Mimetype: mtype,
+				Size:     uint(len(*fi.Data)),
+			},
+		})
 		if err != nil {
 			b.Log.Errorf("sendFile failed: %#v", err)
 		}
 	case strings.Contains(mtype, "audio"):
 		b.Log.Debugf("sendAudio %s", res.ContentURI)
-		_, err = b.mc.SendAudio(channel, fi.Name, res.ContentURI, mtype, uint(len(*fi.Data)))
+		_, err = b.mc.SendMessageEvent(channel, "m.room.message", matrix.AudioMessage{
+			MsgType: "m.audio",
+			Body:    fi.Name,
+			URL:     res.ContentURI,
+			Info: matrix.AudioInfo{
+				Mimetype: mtype,
+				Size:     uint(len(*fi.Data)),
+			},
+		})
 		if err != nil {
 			b.Log.Errorf("sendAudio failed: %#v", err)
 		}
@@ -366,12 +419,39 @@ func (b *Bmatrix) containsAttachment(content map[string]interface{}) bool {
 
 // getAvatarURL returns the avatar URL of the specified sender
 func (b *Bmatrix) getAvatarURL(sender string) string {
-	mxcURL, err := b.mc.GetSenderAvatarURL(sender)
+	urlPath := b.mc.BuildURL("profile", sender, "avatar_url")
+
+	s := struct {
+		AvatarURL string `json:"avatar_url"`
+	}{}
+
+	err := b.mc.MakeRequest("GET", urlPath, nil, &s)
 	if err != nil {
 		b.Log.Errorf("getAvatarURL failed: %s", err)
 		return ""
 	}
-	url := strings.ReplaceAll(mxcURL, "mxc://", b.GetString("Server")+"/_matrix/media/r0/thumbnail/")
-	url += "?width=37&height=37&method=crop"
+	url := strings.ReplaceAll(s.AvatarURL, "mxc://", b.GetString("Server")+"/_matrix/media/r0/thumbnail/")
+	if url != "" {
+		url += "?width=37&height=37&method=crop"
+	}
 	return url
 }
+
+func handleError(err error) *httpError {
+	mErr, ok := err.(matrix.HTTPError)
+	if !ok {
+		return &httpError{
+			Err: "not a HTTPError",
+		}
+	}
+
+	var httpErr httpError
+
+	if err := json.Unmarshal(mErr.Contents, &httpErr); err != nil {
+		return &httpError{
+			Err: "unmarshal failed",
+		}
+	}
+
+	return &httpErr
+}

bridge/mattermost/handlers.go

@@ -4,7 +4,7 @@ import (
 	"github.com/42wim/matterbridge/bridge/config"
 	"github.com/42wim/matterbridge/bridge/helper"
 	"github.com/42wim/matterbridge/matterclient"
-	"github.com/mattermost/mattermost-server/model"
+	"github.com/mattermost/mattermost-server/v5/model"
 )
 
 // handleDownloadAvatar downloads the avatar of userid from channel

bridge/mattermost/helpers.go

@@ -7,7 +7,7 @@ import (
 	"github.com/42wim/matterbridge/bridge/helper"
 	"github.com/42wim/matterbridge/matterclient"
 	"github.com/42wim/matterbridge/matterhook"
-	"github.com/mattermost/mattermost-server/model"
+	"github.com/mattermost/mattermost-server/v5/model"
 )
 
 func (b *Bmattermost) doConnectWebhookBind() error {

bridge/msteams/msteams.go

@@ -10,11 +10,11 @@ import (
 
 	"github.com/42wim/matterbridge/bridge"
 	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/davecgh/go-spew/spew"
 
-	//	"github.com/davecgh/go-spew/spew"
-	"github.com/matterbridge/msgraph.go/msauth"
 	"github.com/mattn/godown"
 	msgraph "github.com/yaegashi/msgraph.go/beta"
+	"github.com/yaegashi/msgraph.go/msauth"
 
 	"golang.org/x/oauth2"
 )
@@ -72,7 +72,15 @@ func (b *Bmsteams) Disconnect() error {
 }
 
 func (b *Bmsteams) JoinChannel(channel config.ChannelInfo) error {
-	go b.poll(channel.Name)
+	go func(name string) {
+		for {
+			err := b.poll(name)
+			if err != nil {
+				b.Log.Errorf("polling failed for %s: %s. retrying in 5 seconds", name, err)
+			}
+			time.Sleep(time.Second * 5)
+		}
+	}(channel.Name)
 	return nil
 }
 
@@ -121,12 +129,12 @@ func (b *Bmsteams) getMessages(channel string) ([]msgraph.ChatMessage, error) {
 }
 
 //nolint:gocognit
-func (b *Bmsteams) poll(channelName string) {
+func (b *Bmsteams) poll(channelName string) error {
 	msgmap := make(map[string]time.Time)
 	b.Log.Debug("getting initial messages")
 	res, err := b.getMessages(channelName)
 	if err != nil {
-		panic(err)
+		return err
 	}
 	for _, msg := range res {
 		msgmap[*msg.ID] = *msg.CreatedDateTime
@@ -139,7 +147,7 @@ func (b *Bmsteams) poll(channelName string) {
 	for {
 		res, err := b.getMessages(channelName)
 		if err != nil {
-			panic(err)
+			return err
 		}
 		for i := len(res) - 1; i >= 0; i-- {
 			msg := res[i]
@@ -151,11 +159,22 @@ func (b *Bmsteams) poll(channelName string) {
 					continue
 				}
 			}
+
+			if b.GetBool("debug") {
+				b.Log.Debug("Msg dump: ", spew.Sdump(msg))
+			}
+
+			// skip non-user message for now.
+			if msg.From.User == nil {
+				continue
+			}
+
 			if *msg.From.User.ID == b.botID {
 				b.Log.Debug("skipping own message")
 				msgmap[*msg.ID] = *msg.CreatedDateTime
 				continue
 			}
+
 			msgmap[*msg.ID] = *msg.CreatedDateTime
 			if msg.LastModifiedDateTime != nil {
 				msgmap[*msg.ID] = *msg.LastModifiedDateTime

bridge/nctalk/nctalk.go

@@ -0,0 +1,146 @@
+package nctalk
+
+import (
+	"context"
+	"crypto/tls"
+	"strconv"
+	"strings"
+
+	"github.com/42wim/matterbridge/bridge"
+	"github.com/42wim/matterbridge/bridge/config"
+
+	talk "gomod.garykim.dev/nc-talk"
+	"gomod.garykim.dev/nc-talk/ocs"
+	"gomod.garykim.dev/nc-talk/room"
+	"gomod.garykim.dev/nc-talk/user"
+)
+
+type Btalk struct {
+	user  *user.TalkUser
+	rooms []Broom
+	*bridge.Config
+}
+
+func New(cfg *bridge.Config) bridge.Bridger {
+	return &Btalk{Config: cfg}
+}
+
+type Broom struct {
+	room      *room.TalkRoom
+	ctx       context.Context
+	ctxCancel context.CancelFunc
+}
+
+func (b *Btalk) Connect() error {
+	b.Log.Info("Connecting")
+	tconfig := &user.TalkUserConfig{
+		TLSConfig: &tls.Config{
+			InsecureSkipVerify: b.GetBool("SkipTLSVerify"), //nolint:gosec
+		},
+	}
+	var err error
+	b.user, err = user.NewUser(b.GetString("Server"), b.GetString("Login"), b.GetString("Password"), tconfig)
+	if err != nil {
+		b.Log.Error("Config could not be used")
+		return err
+	}
+	_, err = b.user.Capabilities()
+	if err != nil {
+		b.Log.Error("Cannot Connect")
+		return err
+	}
+	b.Log.Info("Connected")
+	return nil
+}
+
+func (b *Btalk) Disconnect() error {
+	for _, r := range b.rooms {
+		r.ctxCancel()
+	}
+	return nil
+}
+
+func (b *Btalk) JoinChannel(channel config.ChannelInfo) error {
+	newRoom := Broom{
+		room: talk.NewRoom(b.user, channel.Name),
+	}
+	newRoom.ctx, newRoom.ctxCancel = context.WithCancel(context.Background())
+	c, err := newRoom.room.ReceiveMessages(newRoom.ctx)
+	if err != nil {
+		return err
+	}
+	b.rooms = append(b.rooms, newRoom)
+	go func() {
+		for msg := range c {
+			// ignore messages that are one of the following
+			// * not a message from a user
+			// * from ourselves
+			if msg.MessageType != ocs.MessageComment || msg.ActorID == b.user.User {
+				continue
+			}
+			remoteMessage := config.Message{
+				Text:     formatRichObjectString(msg.Message, msg.MessageParameters),
+				Channel:  newRoom.room.Token,
+				Username: msg.ActorDisplayName,
+				UserID:   msg.ActorID,
+				Account:  b.Account,
+			}
+			// It is possible for the ID to not be set on older versions of Talk so we only set it if
+			// the ID is not blank
+			if msg.ID != 0 {
+				remoteMessage.ID = strconv.Itoa(msg.ID)
+			}
+			b.Log.Debugf("<= Message is %#v", remoteMessage)
+			b.Remote <- remoteMessage
+		}
+	}()
+	return nil
+}
+
+func (b *Btalk) Send(msg config.Message) (string, error) {
+	r := b.getRoom(msg.Channel)
+	if r == nil {
+		b.Log.Errorf("Could not find room for %v", msg.Channel)
+		return "", nil
+	}
+
+	// Talk currently only supports sending normal messages
+	if msg.Event != "" {
+		return "", nil
+	}
+	sentMessage, err := r.room.SendMessage(msg.Username + msg.Text)
+	if err != nil {
+		b.Log.Errorf("Could not send message to room %v from %v: %v", msg.Channel, msg.Username, err)
+		return "", nil
+	}
+	return strconv.Itoa(sentMessage.ID), nil
+}
+
+func (b *Btalk) getRoom(token string) *Broom {
+	for _, r := range b.rooms {
+		if r.room.Token == token {
+			return &r
+		}
+	}
+	return nil
+}
+
+// Spec: https://github.com/nextcloud/server/issues/1706#issue-182308785
+func formatRichObjectString(message string, parameters map[string]ocs.RichObjectString) string {
+	for id, parameter := range parameters {
+		text := parameter.Name
+
+		switch parameter.Type {
+		case ocs.ROSTypeUser, ocs.ROSTypeGroup:
+			text = "@" + text
+		case ocs.ROSTypeFile:
+			if parameter.Link != "" {
+				text = parameter.Link
+			}
+		}
+
+		message = strings.ReplaceAll(message, "{"+id+"}", text)
+	}
+
+	return message
+}

bridge/rocketchat/handlers.go

@@ -2,6 +2,7 @@ package brocketchat
 
 import (
 	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/matterbridge/Rocket.Chat.Go.SDK/models"
 )
 
 func (b *Brocketchat) handleRocket() {
@@ -38,6 +39,23 @@ func (b *Brocketchat) handleRocketHook(messages chan *config.Message) {
 	}
 }
 
+func (b *Brocketchat) handleStatusEvent(ev models.Message, rmsg *config.Message) bool {
+	switch ev.Type {
+	case "":
+		// this is a normal message, no processing needed
+		// return true so the message is not dropped
+		return true
+	case sUserJoined, sUserLeft:
+		rmsg.Event = config.EventJoinLeave
+		return true
+	case sRoomChangedTopic:
+		rmsg.Event = config.EventTopicChange
+		return true
+	}
+	b.Log.Debugf("Dropping message with unknown type: %s", ev.Type)
+	return false
+}
+
 func (b *Brocketchat) handleRocketClient(messages chan *config.Message) {
 	for message := range b.messageChan {
 		// skip messages with same ID, apparently messages get duplicated for an unknown reason
@@ -59,7 +77,12 @@ func (b *Brocketchat) handleRocketClient(messages chan *config.Message) {
 			UserID:   message.User.ID,
 			ID:       message.ID,
 		}
-		messages <- rmsg
+
+		// handleStatusEvent returns false if the message should be dropped
+		// in that case it is probably some modification to the channel we do not want to relay
+		if b.handleStatusEvent(m, rmsg) {
+			messages <- rmsg
+		}
 	}
 }
 

bridge/rocketchat/rocketchat.go

@@ -29,6 +29,12 @@ type Brocketchat struct {
 	sync.RWMutex
 }
 
+const (
+	sUserJoined       = "uj"
+	sUserLeft         = "ul"
+	sRoomChangedTopic = "room_changed_topic"
+)
+
 func New(cfg *bridge.Config) bridge.Bridger {
 	newCache, err := lru.New(100)
 	if err != nil {

bridge/slack/handlers.go

@@ -16,7 +16,7 @@ var ErrEventIgnored = errors.New("this event message should ignored")
 
 func (b *Bslack) handleSlack() {
 	messages := make(chan *config.Message)
-	if b.GetString(incomingWebhookConfig) != "" {
+	if b.GetString(incomingWebhookConfig) != "" && b.GetString(tokenConfig) == "" {
 		b.Log.Debugf("Choosing webhooks based receiving")
 		go b.handleMatterHook(messages)
 	} else {
@@ -137,12 +137,6 @@ func (b *Bslack) skipMessageEvent(ev *slack.MessageEvent) bool {
 		hasOurCallbackID = ok && block.BlockID == "matterbridge_"+b.uuid
 	}
 
-	// Skip any messages that we made ourselves or from 'slackbot' (see #527).
-	if ev.Username == sSlackBotUser ||
-		(b.rtm != nil && ev.Username == b.si.User.Name) || hasOurCallbackID {
-		return true
-	}
-
 	if ev.SubMessage != nil {
 		// It seems ev.SubMessage.Edited == nil when slack unfurls.
 		// Do not forward these messages. See Github issue #266.
@@ -155,6 +149,16 @@ func (b *Bslack) skipMessageEvent(ev *slack.MessageEvent) bool {
 		if ev.SubType == "message_replied" && ev.Hidden {
 			return true
 		}
+		if len(ev.SubMessage.Blocks.BlockSet) == 1 {
+			block, ok := ev.SubMessage.Blocks.BlockSet[0].(*slack.SectionBlock)
+			hasOurCallbackID = ok && block.BlockID == "matterbridge_"+b.uuid
+		}
+	}
+
+	// Skip any messages that we made ourselves or from 'slackbot' (see #527).
+	if ev.Username == sSlackBotUser ||
+		(b.rtm != nil && ev.Username == b.si.User.Name) || hasOurCallbackID {
+		return true
 	}
 
 	if len(ev.Files) > 0 {

bridge/slack/slack.go

@@ -64,6 +64,7 @@ const (
 	editSuffixConfig      = "EditSuffix"
 	iconURLConfig         = "iconurl"
 	noSendJoinConfig      = "nosendjoinpart"
+	messageLength         = 3000
 )
 
 func New(cfg *bridge.Config) bridge.Bridger {
@@ -194,6 +195,7 @@ func (b *Bslack) Send(msg config.Message) (string, error) {
 		b.Log.Debugf("=> Receiving %#v", msg)
 	}
 
+	msg.Text = helper.ClipMessage(msg.Text, messageLength)
 	msg.Text = b.replaceCodeFence(msg.Text)
 
 	// Make a action /me of the message
@@ -202,7 +204,7 @@ func (b *Bslack) Send(msg config.Message) (string, error) {
 	}
 
 	// Use webhook to send the message
-	if b.GetString(outgoingWebhookConfig) != "" {
+	if b.GetString(outgoingWebhookConfig) != "" && b.GetString(tokenConfig) == "" {
 		return "", b.sendWebhook(msg)
 	}
 	return b.sendRTM(msg)

bridge/telegram/handlers.go

@@ -39,22 +39,32 @@ func (b *Btelegram) handleGroups(rmsg *config.Message, message *tgbotapi.Message
 
 // handleForwarded handles forwarded messages
 func (b *Btelegram) handleForwarded(rmsg *config.Message, message *tgbotapi.Message) {
-	if message.ForwardFrom != nil {
-		usernameForward := ""
-		if b.GetBool("UseFirstName") {
+	if message.ForwardDate == 0 {
+		return
+	}
+
+	if message.ForwardFrom == nil {
+		rmsg.Text = "Forwarded from " + unknownUser + ": " + rmsg.Text
+		return
+	}
+
+	usernameForward := ""
+	if b.GetBool("UseFirstName") {
+		usernameForward = message.ForwardFrom.FirstName
+	}
+
+	if usernameForward == "" {
+		usernameForward = message.ForwardFrom.UserName
+		if usernameForward == "" {
 			usernameForward = message.ForwardFrom.FirstName
 		}
-		if usernameForward == "" {
-			usernameForward = message.ForwardFrom.UserName
-			if usernameForward == "" {
-				usernameForward = message.ForwardFrom.FirstName
-			}
-		}
-		if usernameForward == "" {
-			usernameForward = unknownUser
-		}
-		rmsg.Text = "Forwarded from " + usernameForward + ": " + rmsg.Text
 	}
+
+	if usernameForward == "" {
+		usernameForward = unknownUser
+	}
+
+	rmsg.Text = "Forwarded from " + usernameForward + ": " + rmsg.Text
 }
 
 // handleQuoting handles quoting of previous messages
@@ -207,6 +217,46 @@ func (b *Btelegram) handleDownloadAvatar(userid int, channel string) {
 	}
 }
 
+func (b *Btelegram) maybeConvertTgs(name *string, data *[]byte) {
+	var format string
+	switch b.GetString("MediaConvertTgs") {
+	case FormatWebp:
+		b.Log.Debugf("Tgs to WebP conversion enabled, converting %v", name)
+		format = FormatWebp
+	case FormatPng:
+		// The WebP to PNG converter can't handle animated webp files yet,
+		// and I'm not going to write a path for x/image/webp.
+		// The error message would be:
+		//     conversion failed: webp: non-Alpha VP8X is not implemented
+		// So instead, we tell lottie to directly go to PNG.
+		b.Log.Debugf("Tgs to PNG conversion enabled, converting %v", name)
+		format = FormatPng
+	default:
+		// Otherwise, no conversion was requested. Trying to run the usual webp
+		// converter would fail, because '.tgs.webp' is actually a gzipped JSON
+		// file, and has nothing to do with WebP.
+		return
+	}
+	err := helper.ConvertTgsToX(data, format, b.Log)
+	if err != nil {
+		b.Log.Errorf("conversion failed: %v", err)
+	} else {
+		*name = strings.Replace(*name, "tgs.webp", format, 1)
+	}
+}
+
+func (b *Btelegram) maybeConvertWebp(name *string, data *[]byte) {
+	if b.GetBool("MediaConvertWebPToPNG") {
+		b.Log.Debugf("WebP to PNG conversion enabled, converting %v", name)
+		err := helper.ConvertWebPToPNG(data)
+		if err != nil {
+			b.Log.Errorf("conversion failed: %v", err)
+		} else {
+			*name = strings.Replace(*name, ".webp", ".png", 1)
+		}
+	}
+}
+
 // handleDownloadFile handles file download
 func (b *Btelegram) handleDownload(rmsg *config.Message, message *tgbotapi.Message) error {
 	size := 0
@@ -254,15 +304,13 @@ func (b *Btelegram) handleDownload(rmsg *config.Message, message *tgbotapi.Messa
 	if err != nil {
 		return err
 	}
-	if strings.HasSuffix(name, ".webp") && b.GetBool("MediaConvertWebPToPNG") {
-		b.Log.Debugf("WebP to PNG conversion enabled, converting %s", name)
-		err := helper.ConvertWebPToPNG(data)
-		if err != nil {
-			b.Log.Errorf("conversion failed: %s", err)
-		} else {
-			name = strings.Replace(name, ".webp", ".png", 1)
-		}
+
+	if strings.HasSuffix(name, ".tgs.webp") {
+		b.maybeConvertTgs(&name, data)
+	} else if strings.HasSuffix(name, ".webp") {
+		b.maybeConvertWebp(&name, data)
 	}
+
 	helper.HandleDownloadData(b.Log, rmsg, name, message.Caption, "", data, b.General)
 	return nil
 }
@@ -312,6 +360,9 @@ func (b *Btelegram) handleEdit(msg *config.Message, chatid int64) (string, error
 	case "Markdown":
 		b.Log.Debug("Using mode markdown")
 		m.ParseMode = tgbotapi.ModeMarkdown
+	case MarkdownV2:
+		b.Log.Debug("Using mode MarkdownV2")
+		m.ParseMode = MarkdownV2
 	}
 	if strings.ToLower(b.GetString("MessageFormat")) == HTMLNick {
 		b.Log.Debug("Using mode HTML - nick only")

bridge/telegram/telegram.go

@@ -2,6 +2,7 @@ package btelegram
 
 import (
 	"html"
+	"log"
 	"strconv"
 	"strings"
 
@@ -15,6 +16,9 @@ const (
 	unknownUser = "unknown"
 	HTMLFormat  = "HTML"
 	HTMLNick    = "htmlnick"
+	MarkdownV2  = "MarkdownV2"
+	FormatPng   = "png"
+	FormatWebp  = "webp"
 )
 
 type Btelegram struct {
@@ -24,6 +28,16 @@ type Btelegram struct {
 }
 
 func New(cfg *bridge.Config) bridge.Bridger {
+	tgsConvertFormat := cfg.GetString("MediaConvertTgs")
+	if tgsConvertFormat != "" {
+		err := helper.CanConvertTgsToX()
+		if err != nil {
+			log.Fatalf("Telegram bridge configured to convert .tgs files to '%s', but lottie does not appear to work:\n%#v", tgsConvertFormat, err)
+		}
+		if tgsConvertFormat != FormatPng && tgsConvertFormat != FormatWebp {
+			log.Fatalf("Telegram bridge configured to convert .tgs files to '%s', but only '%s' and '%s' are supported.", FormatPng, FormatWebp, tgsConvertFormat)
+		}
+	}
 	return &Btelegram{Config: cfg, avatarMap: make(map[string]string)}
 }
 
@@ -126,6 +140,10 @@ func (b *Btelegram) sendMessage(chatid int64, username, text string) (string, er
 		b.Log.Debug("Using mode markdown")
 		m.ParseMode = tgbotapi.ModeMarkdown
 	}
+	if b.GetString("MessageFormat") == MarkdownV2 {
+		b.Log.Debug("Using mode MarkdownV2")
+		m.ParseMode = MarkdownV2
+	}
 	if strings.ToLower(b.GetString("MessageFormat")) == HTMLNick {
 		b.Log.Debug("Using mode HTML - nick only")
 		m.Text = username + html.EscapeString(text)

bridge/whatsapp/handlers.go

@@ -23,7 +23,8 @@ Check:
 // HandleError received from WhatsApp
 func (b *Bwhatsapp) HandleError(err error) {
 	// ignore received invalid data errors. https://github.com/42wim/matterbridge/issues/843
-	if strings.Contains(err.Error(), "error processing data: received invalid data") {
+	// ignore tag 174 errors. https://github.com/42wim/matterbridge/issues/1094
+	if strings.Contains(err.Error(), "error processing data: received invalid data") || strings.Contains(err.Error(), "invalid string with tag 174") {
 		return
 	}
 
@@ -55,6 +56,7 @@ func (b *Bwhatsapp) reconnect(err error) {
 		err := b.conn.Restore()
 		if err == nil {
 			bf.Reset()
+			b.startedAt = uint64(time.Now().Unix())
 			return
 		}
 	}
@@ -76,7 +78,9 @@ func (b *Bwhatsapp) HandleTextMessage(message whatsapp.TextMessage) {
 	senderJID := message.Info.SenderJid
 	if len(senderJID) == 0 {
 		// TODO workaround till https://github.com/Rhymen/go-whatsapp/issues/86 resolved
-		senderJID = *message.Info.Source.Participant
+		if message.Info.Source != nil && message.Info.Source.Participant != nil {
+			senderJID = *message.Info.Source.Participant
+		}
 	}
 
 	// translate sender's JID to the nicest username we can get

bridge/whatsapp/helpers.go

@@ -80,8 +80,33 @@ func (b *Bwhatsapp) getSenderName(senderJid string) string {
 		// if user is not in phone contacts
 		// it is the most obvious scenario unless you sync your phone contacts with some remote updated source
 		// users can change it in their WhatsApp settings -> profile -> click on Avatar
-		return sender.Notify
+		if sender.Notify != "" {
+			return sender.Notify
+		}
+
+		if sender.Short != "" {
+			return sender.Short
+		}
 	}
+
+	// try to reload this contact
+	_, err := b.conn.Contacts()
+	if err != nil {
+		b.Log.Errorf("error on update of contacts: %v", err)
+	}
+
+	if contact, exists := b.conn.Store.Contacts[senderJid]; exists {
+		// Add it to the user map
+		b.users[senderJid] = contact
+
+		if contact.Name != "" {
+			return contact.Name
+		}
+		// if user is not in phone contacts
+		// same as above
+		return contact.Notify
+	}
+
 	return ""
 }
 

bridge/whatsapp/whatsapp.go

@@ -67,7 +67,7 @@ func (b *Bwhatsapp) Connect() error {
 	// https://github.com/Rhymen/go-whatsapp#creating-a-connection
 	b.Log.Debugln("Connecting to WhatsApp..")
 	conn, err := whatsapp.NewConn(20 * time.Second)
-	conn.SetClientVersion(0, 4, 1307)
+	conn.SetClientVersion(0, 4, 2080)
 	if err != nil {
 		return errors.New("failed to connect to WhatsApp: " + err.Error())
 	}
@@ -184,7 +184,7 @@ func (b *Bwhatsapp) Disconnect() error {
 }
 
 func isGroupJid(identifier string) bool {
-	return strings.HasSuffix(identifier, "@g.us") || strings.HasSuffix(identifier, "@temp")
+	return strings.HasSuffix(identifier, "@g.us") || strings.HasSuffix(identifier, "@temp") || strings.HasSuffix(identifier, "@broadcast")
 }
 
 // JoinChannel Join a WhatsApp group specified in gateway config as channel='number-id@g.us' or channel='Channel name'

bridge/xmpp/handler.go

@@ -0,0 +1,34 @@
+package bxmpp
+
+import (
+	"github.com/42wim/matterbridge/bridge/config"
+	"github.com/42wim/matterbridge/bridge/helper"
+	"github.com/matterbridge/go-xmpp"
+)
+
+// handleDownloadAvatar downloads the avatar of userid from channel
+// sends a EVENT_AVATAR_DOWNLOAD message to the gateway if successful.
+// logs an error message if it fails
+func (b *Bxmpp) handleDownloadAvatar(avatar xmpp.AvatarData) {
+	rmsg := config.Message{
+		Username: "system",
+		Text:     "avatar",
+		Channel:  b.parseChannel(avatar.From),
+		Account:  b.Account,
+		UserID:   avatar.From,
+		Event:    config.EventAvatarDownload,
+		Extra:    make(map[string][]interface{}),
+	}
+	if _, ok := b.avatarMap[avatar.From]; !ok {
+		b.Log.Debugf("Avatar.From: %s", avatar.From)
+
+		err := helper.HandleDownloadSize(b.Log, &rmsg, avatar.From+".png", int64(len(avatar.Data)), b.General)
+		if err != nil {
+			b.Log.Error(err)
+			return
+		}
+		helper.HandleDownloadData(b.Log, &rmsg, avatar.From+".png", rmsg.Text, "", &avatar.Data, b.General)
+		b.Log.Debugf("Avatar download complete")
+		b.Remote <- rmsg
+	}
+}

bridge/xmpp/helpers.go

@@ -0,0 +1,30 @@
+package bxmpp
+
+import (
+	"regexp"
+
+	"github.com/42wim/matterbridge/bridge/config"
+)
+
+var pathRegex = regexp.MustCompile("[^a-zA-Z0-9]+")
+
+// GetAvatar constructs a URL for a given user-avatar if it is available in the cache.
+func getAvatar(av map[string]string, userid string, general *config.Protocol) string {
+	if hash, ok := av[userid]; ok {
+		// NOTE: This does not happen in bridge/helper/helper.go but messes up XMPP
+		id := pathRegex.ReplaceAllString(userid, "_")
+		return general.MediaServerDownload + "/" + hash + "/" + id + ".png"
+	}
+	return ""
+}
+
+func (b *Bxmpp) cacheAvatar(msg *config.Message) string {
+	fi := msg.Extra["file"][0].(config.FileInfo)
+	/* if we have a sha we have successfully uploaded the file to the media server,
+	so we can now cache the sha */
+	if fi.SHA != "" {
+		b.Log.Debugf("Added %s to %s in avatarMap", fi.SHA, msg.UserID)
+		b.avatarMap[msg.UserID] = fi.SHA
+	}
+	return ""
+}

bridge/xmpp/xmpp.go

@@ -23,12 +23,17 @@ type Bxmpp struct {
 	xmppMap   map[string]string
 	connected bool
 	sync.RWMutex
+
+	avatarAvailability map[string]bool
+	avatarMap          map[string]string
 }
 
 func New(cfg *bridge.Config) bridge.Bridger {
 	return &Bxmpp{
-		Config:  cfg,
-		xmppMap: make(map[string]string),
+		Config:             cfg,
+		xmppMap:            make(map[string]string),
+		avatarAvailability: make(map[string]bool),
+		avatarMap:          make(map[string]string),
 	}
 }
 
@@ -67,8 +72,19 @@ func (b *Bxmpp) Send(msg config.Message) (string, error) {
 	if msg.Event == config.EventMsgDelete {
 		return "", nil
 	}
+
 	b.Log.Debugf("=> Receiving %#v", msg)
 
+	if msg.Event == config.EventAvatarDownload {
+		return b.cacheAvatar(&msg), nil
+	}
+
+	// Make a action /me of the message, prepend the username with it.
+	// https://xmpp.org/extensions/xep-0245.html
+	if msg.Event == config.EventUserAction {
+		msg.Username = "/me " + msg.Username
+	}
+
 	// Upload a file (in XMPP case send the upload URL because XMPP has no native upload support).
 	if msg.Extra != nil {
 		for _, rmsg := range helper.HandleExtra(&msg, b.General) {
@@ -114,6 +130,9 @@ func (b *Bxmpp) createXMPP() error {
 		ServerName:         strings.Split(b.GetString("Jid"), "@")[1],
 		InsecureSkipVerify: b.GetBool("SkipTLSVerify"), // nolint: gosec
 	}
+
+	xmpp.DebugWriter = b.Log.Writer()
+
 	options := xmpp.Options{
 		Host:                         b.GetString("Server"),
 		User:                         b.GetString("Jid"),
@@ -122,7 +141,6 @@ func (b *Bxmpp) createXMPP() error {
 		StartTLS:                     true,
 		TLSConfig:                    tc,
 		Debug:                        b.GetBool("debug"),
-		Logger:                       b.Log.Writer(),
 		Session:                      true,
 		Status:                       "",
 		StatusMessage:                "",
@@ -228,6 +246,16 @@ func (b *Bxmpp) handleXMPP() error {
 					event = config.EventTopicChange
 				}
 
+				available, sok := b.avatarAvailability[v.Remote]
+				avatar := ""
+				if !sok {
+					b.Log.Debugf("Requesting avatar data")
+					b.avatarAvailability[v.Remote] = false
+					b.xc.AvatarRequestData(v.Remote)
+				} else if available {
+					avatar = getAvatar(b.avatarMap, v.Remote, b.General)
+				}
+
 				msgID := v.ID
 				if v.ReplaceID != "" {
 					msgID = v.ReplaceID
@@ -237,6 +265,7 @@ func (b *Bxmpp) handleXMPP() error {
 					Text:     v.Text,
 					Channel:  b.parseChannel(v.Remote),
 					Account:  b.Account,
+					Avatar:   avatar,
 					UserID:   v.Remote,
 					ID:       msgID,
 					Event:    event,
@@ -253,6 +282,10 @@ func (b *Bxmpp) handleXMPP() error {
 				b.Log.Debugf("<= Message is %#v", rmsg)
 				b.Remote <- rmsg
 			}
+		case xmpp.AvatarData:
+			b.handleDownloadAvatar(v)
+			b.avatarAvailability[v.From] = true
+			b.Log.Debugf("Avatar for %s is now available", v.From)
 		case xmpp.Presence:
 			// Do nothing.
 		}

bridge/zulip/zulip.go

@@ -135,19 +135,25 @@ func (b *Bzulip) handleQueue() error {
 			if m.SenderEmail == b.GetString("login") {
 				continue
 			}
+
+			avatarURL := m.AvatarURL
+			if !strings.HasPrefix(avatarURL, "http") {
+				avatarURL = b.GetString("server") + avatarURL
+			}
+
 			rmsg := config.Message{
 				Username: m.SenderFullName,
 				Text:     m.Content,
 				Channel:  b.getChannel(m.StreamID) + "/topic:" + m.Subject,
 				Account:  b.Account,
 				UserID:   strconv.Itoa(m.SenderID),
-				Avatar:   m.AvatarURL,
+				Avatar:   avatarURL,
 			}
 			b.Log.Debugf("<= Sending message from %s on %s to gateway", rmsg.Username, b.Account)
 			b.Log.Debugf("<= Message is %#v", rmsg)
 			b.Remote <- rmsg
-			b.q.LastEventID = m.ID
 		}
+
 		time.Sleep(time.Second * 3)
 	}
 }

changelog.md

@@ -1,3 +1,147 @@
+# v1.18.3
+
+## Enhancements
+
+- nctalk: Add TLSConfig to nctalk (#1195)
+- whatsapp: Handle broadcasts as groups in Whatsapp #1213
+- matrix: switch to upstream gomatrix #1219
+- api: support multiple websocket clients #1205
+
+## Bugfix
+
+- general: update vendor
+- zulip: Check location of avatarURL (zulip). Fixes #1214 (#1227)
+- nctalk: Fix issue with too many open files #1223
+- nctalk: Fix mentions #1222
+- nctalk: Fix message replays #1220
+
+This release couldn't exist without the following contributors:
+@gary-kim, @tilosp, @NikkyAI, @escoand, @42wim
+
+# v1.18.2
+
+## Bugfix
+
+- zulip: Fix error loop (zulip) (#1210)
+- whatsapp: Update whatsapp vendor and fix a panic (#1209)
+
+This release couldn't exist without the following contributors:
+@SuperSandro2000, @42wim
+
+# v1.18.1
+
+## New features
+
+- telegram: Support Telegram animated stickers (tgs) format (#1173). See https://github.com/42wim/matterbridge/wiki/Settings#mediaConverttgs for more info
+
+## Enhancements
+
+- matrix: Remove HTML formatting for push messages (#1188) (#1189)
+- mattermost: Use mattermost v5 module (#1192)
+
+## Bugfix
+
+- whatsapp: Handle panic in whatsapp. Fixes #1180 (#1184)
+- nctalk: Fix Nextcloud Talk connection failure (#1179)
+- matrix: Sleep when ratelimited on joins (matrix). Fixes #1201 (#1206)
+
+This release couldn't exist without the following contributors:
+@42wim, @BenWiederhake, @Dellle, @gary-kim
+
+# v1.18.0
+
+## New features
+
+- nctalk: new protocol added. Add Nextcloud Talk support #1167
+- general: Add an option to log into a file rather than stdout (#1168)
+- api: Add websocket to API (#970)
+
+## Enhancements
+
+- telegram: Fix MarkdownV2 support in Telegram (#1169)
+- whatsapp: Reload user information when a new contact is detected (whatsapp) (#1160)
+- api: Add sane RemoteNickFormat default for API (#1157)
+- irc: Skip gIRC built-in rate limiting (irc) (#1164)
+- irc: Only colour IRC nicks if there is one. (#1161)
+- docker: Combine runs to one layer (#1151)
+
+## Bugfix
+
+- general: Update dependencies for 1.18.0 release (#1175)
+
+Discord users are encouraged to upgrade, this release works with the move to the discord.com domain.
+
+This release couldn't exist without the following contributors:
+@42wim, @jlu5, @qaisjp, @TheHolyRoger, @SuperSandro2000, @gary-kim, @z3bra, @greenx, @haykam821, @nathanaelhoun
+
+# v1.17.5
+
+## Enhancements
+
+- irc: Add StripMarkdown option (irc). (#1145)
+- general: Increase debug logging with function,file and linenumber (#1147)
+- general: Update Dockerfile so inotify works (#1148)
+- matrix: Add an option to disable sending HTML to matrix. Fixes #1022 (#1135)
+- xmpp: Implement xep-0245 (xmpp). Closes #1137 (#1144)
+
+## Bugfix
+
+- discord: Fix #1120: replaceAction "_" crash (discord) (#1121)
+- discord: Fix #1049: missing space before embeds (discord) (#1124)
+- discord: Fix webhook EventUserAction messages being skipped (discord) (#1133)
+- matrix: Avoid creating invalid url when the user doesn't have an avatar (matrix) (#1130)
+- msteams: Ignore non-user messages (msteams). Fixes #1141 (#1149)
+- slack: Do not use webhooks when token is configured (slack) (fixes #1123) (#1134)
+- telegram: Fix forward from hidden users (telegram). Closes #1131 (#1143)
+- xmpp: Prevent re-requesting avatar data (xmpp) (#1117)
+
+This release couldn't exist without the following contributors:
+@qaisjp, @xnaas, @42wim, @Polynomdivision, @tfve
+
+# v1.17.4
+
+## Bugfix
+
+- general: Lowercase account names. Fixes #1108 (#1110)
+- msteams: Remove panics and retry polling on failure (msteams). Fixes #1104 (#1105
+- whatsapp: Update Rhymen/go-whatsapp. Fixes #1107 (#1109) (make whatsapp working again)
+- discord: Add an ID cache (discord). Fixes #1106 (#1111) (fix delete/edits with webhooks)
+
+# v1.17.3
+
+## Enhancements
+
+- xmpp: Implement User Avatar spoofing of XMPP users #1090
+- rocketchat: Relay Joins/Topic changes in RocketChat bridge (#1085)
+- irc: Add JoinDelay option (irc). Fixes #1084 (#1098)
+- slack: Clip too long messages on 3000 length (slack). Fixes #1081 (#1102)
+
+## Bugfix
+
+- general: Fix the behavior of ShowTopicChange and SyncTopic (#1086)
+- slack: Prevent image/message looping (slack). Fixes #1088 (#1096)
+- whatsapp: Ignore non-critical errors (whatsapp). Fixes #1094 (#1100)
+- irc: Add extra space before colon in attachments (irc). Fixes #1089 (#1101)
+
+This release couldn't exist without the following contributors:
+@42wim, @ldruschk, @qaisjp, @Polynomdivision
+
+# v1.17.2
+
+## Enhancements
+
+- slack: Update vendor slack-go/slack (#1068)
+- general: Update vendor d5/tengo (#1066)
+- general: Clarify terminology used in mapping group chat IDs to channels in config (#1079)
+
+## Bugfix
+
+- whatsapp: Update Rhymen/go-whatsapp vendor and whatsapp version (#1078). Fixes Media upload #1074
+- whatsapp: Reset start timestamp on reconnect (whatsapp). Fixes #1059 (#1064)
+
+This release couldn't exist without the following contributors:
+@42wim, @jheiselman
+
 # v1.17.1
 
 ## Enhancements

ci/bintray.sh

@@ -1,30 +0,0 @@
-#!/usr/bin/env bash
-set -u -e -x -o pipefail
-
-go version | grep go1.14 || exit
-
-VERSION=$(git describe --tags)
-mkdir ci/binaries
-GOOS=windows GOARCH=amd64 go build -ldflags "-s -w -X main.githash=$(git log --pretty=format:'%h' -n 1)" -o ci/binaries/matterbridge-$VERSION-windows-amd64.exe
-GOOS=linux GOARCH=amd64 go build -ldflags "-s -w -X main.githash=$(git log --pretty=format:'%h' -n 1)" -o ci/binaries/matterbridge-$VERSION-linux-amd64
-GOOS=linux GOARCH=arm go build -ldflags "-s -w -X main.githash=$(git log --pretty=format:'%h' -n 1)" -o ci/binaries/matterbridge-$VERSION-linux-arm
-GOOS=darwin GOARCH=amd64 go build -ldflags "-s -w -X main.githash=$(git log --pretty=format:'%h' -n 1)" -o ci/binaries/matterbridge-$VERSION-darwin-amd64
-cd ci
-cat > deploy.json <<EOF
-{
-    "package": {
-        "name": "Matterbridge",
-        "repo": "nightly",
-        "subject": "42wim"
-    },
-    "version": {
-        "name": "$VERSION"
-    },
-    "files":
-        [
-        {"includePattern": "ci/binaries/(.*)", "uploadPattern":"\$1"}
-        ],
-    "publish": true
-}
-EOF
-

ci/lint.sh

@@ -1,17 +0,0 @@
-#!/usr/bin/env bash
-set -u -e -x -o pipefail
-
-if [[ -n "${GOLANGCI_VERSION-}" ]]; then
-  # Retrieve the golangci-lint linter binary.
-  curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | bash -s -- -b ${GOPATH}/bin ${GOLANGCI_VERSION}
-fi
-
-# Run the linter.
-golangci-lint run
-
-# if [[ "${GO111MODULE-off}" == "on" ]]; then
-#   # If Go modules are active then check that dependencies are correctly maintained.
-#   go mod tidy
-#   go mod vendor
-#   git diff --exit-code --quiet || (echo "Please run 'go mod tidy' to clean up the 'go.mod' and 'go.sum' files."; false)
-# fi

ci/test.sh

@@ -1,17 +0,0 @@
-#!/usr/bin/env bash
-set -u -e -x -o pipefail
-
-if [[ -n "${REPORT_COVERAGE+cover}" ]]; then
-  # Retrieve and prepare CodeClimate's test coverage reporter.
-  curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
-  chmod +x ./cc-test-reporter
-  ./cc-test-reporter before-build
-fi
-
-# Run all the tests with the race detector and generate coverage.
-go test -v -race -coverprofile c.out ./...
-
-if [[ -n "${REPORT_COVERAGE+cover}" && "${TRAVIS_SECURE_ENV_VARS}" == "true" ]]; then
-  # Upload test coverage to CodeClimate.
-  ./cc-test-reporter after-build
-fi

gateway/bridgemap/bnctalk.go

@@ -0,0 +1,11 @@
+// +build !nonctalk
+
+package bridgemap
+
+import (
+	btalk "github.com/42wim/matterbridge/bridge/nctalk"
+)
+
+func init() {
+	FullMap["nctalk"] = btalk.New
+}

gateway/gateway.go

@@ -108,7 +108,7 @@ func (gw *Gateway) AddBridge(cfg *config.Bridge) error {
 func (gw *Gateway) checkConfig(cfg *config.Bridge) {
 	match := false
 	for _, key := range gw.Router.Config.Viper().AllKeys() {
-		if strings.HasPrefix(key, cfg.Account) {
+		if strings.HasPrefix(key, strings.ToLower(cfg.Account)) {
 			match = true
 			break
 		}

gateway/handlers.go

@@ -169,7 +169,7 @@ func (gw *Gateway) ignoreEvent(event string, dest *bridge.Bridge) bool {
 	switch event {
 	case config.EventAvatarDownload:
 		// Avatar downloads are only relevant for telegram and mattermost for now
-		if dest.Protocol != "mattermost" && dest.Protocol != "telegram" {
+		if dest.Protocol != "mattermost" && dest.Protocol != "telegram" && dest.Protocol != "xmpp" {
 			return true
 		}
 	case config.EventJoinLeave:
@@ -179,7 +179,7 @@ func (gw *Gateway) ignoreEvent(event string, dest *bridge.Bridge) bool {
 		}
 	case config.EventTopicChange:
 		// only relay topic change when used in some way on other side
-		if dest.GetBool("ShowTopicChange") && dest.GetBool("SyncTopic") {
+		if !dest.GetBool("ShowTopicChange") && !dest.GetBool("SyncTopic") {
 			return true
 		}
 	}

go.mod

@@ -5,61 +5,52 @@ require (
 	github.com/Baozisoftware/qrcode-terminal-go v0.0.0-20170407111555-c0650d8dff0f
 	github.com/Jeffail/gabs v1.1.1 // indirect
 	github.com/Philipp15b/go-steam v1.0.1-0.20190816133340-b04c5a83c1c0
-	github.com/Rhymen/go-whatsapp v0.1.0
-	github.com/d5/tengo/v2 v2.0.2
-	github.com/dfordsoft/golib v0.0.0-20180902042739-76ee6ab99bec
-	github.com/fsnotify/fsnotify v1.4.7
-	github.com/go-telegram-bot-api/telegram-bot-api v4.6.5-0.20181225215658-ec221ba9ea45+incompatible
-	github.com/gomarkdown/markdown v0.0.0-20200127000047-1813ea067497
-	github.com/google/gops v0.3.6
+	github.com/Rhymen/go-whatsapp v0.1.1-0.20200818115958-f07a700b9819
+	github.com/d5/tengo/v2 v2.6.0
+	github.com/davecgh/go-spew v1.1.1
+	github.com/fsnotify/fsnotify v1.4.9
+	github.com/go-telegram-bot-api/telegram-bot-api v1.0.1-0.20200524105306-7434b0456e81
+	github.com/gomarkdown/markdown v0.0.0-20200824053859-8c8b3816f167
+	github.com/google/gops v0.3.11
 	github.com/gopackage/ddp v0.0.0-20170117053602-652027933df4 // indirect
-	github.com/gorilla/schema v1.1.0
-	github.com/gorilla/websocket v1.4.1
-	github.com/hashicorp/golang-lru v0.5.3
-	github.com/hpcloud/tail v1.0.0 // indirect
+	github.com/gorilla/schema v1.2.0
+	github.com/gorilla/websocket v1.4.2
+	github.com/hashicorp/golang-lru v0.5.4
 	github.com/jpillora/backoff v1.0.0
-	github.com/keybase/go-keybase-chat-bot v0.0.0-20200226211841-4e48f3eaef3e
-	github.com/labstack/echo/v4 v4.1.13
+	github.com/keybase/go-keybase-chat-bot v0.0.0-20200505163032-5cacf52379da
+	github.com/labstack/echo/v4 v4.1.17
 	github.com/lrstanley/girc v0.0.0-20190801035559-4fc93959e1a7
-	github.com/matterbridge/Rocket.Chat.Go.SDK v0.0.0-20190210153444-cc9d05784d5d
-	github.com/matterbridge/discordgo v0.18.1-0.20200308151012-aa40f01cbcc3
+	github.com/matrix-org/gomatrix v0.0.0-20200827122206-7dd5e2a05bcd
+	github.com/matterbridge/Rocket.Chat.Go.SDK v0.0.0-20200411204219-d5c18ce75048
+	github.com/matterbridge/discordgo v0.22.0
 	github.com/matterbridge/emoji v2.1.1-0.20191117213217-af507f6b02db+incompatible
-	github.com/matterbridge/go-xmpp v0.0.0-20180529212104-cd19799fba91
-	github.com/matterbridge/gomatrix v0.0.0-20200209224845-c2104d7936a6
-	github.com/matterbridge/gozulipbot v0.0.0-20190212232658-7aa251978a18
-	github.com/matterbridge/logrus-prefixed-formatter v0.0.0-20180806162718-01618749af61
-	github.com/matterbridge/msgraph.go v0.0.0-20200308150230-9e043fe9dbaa
-	github.com/mattermost/mattermost-server v5.5.0+incompatible
-	github.com/mattn/go-runewidth v0.0.7 // indirect
-	github.com/mattn/godown v0.0.0-20180312012330-2e9e17e0ea51
+	github.com/matterbridge/go-xmpp v0.0.0-20200418225040-c8a3a57b4050
+	github.com/matterbridge/gozulipbot v0.0.0-20200820220548-be5824faa913
+	github.com/matterbridge/logrus-prefixed-formatter v0.5.3-0.20200523233437-d971309a77ba
+	github.com/mattermost/mattermost-server/v5 v5.25.2
+	github.com/mattn/godown v0.0.0-20200217152941-afc959f6a561
 	github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect
+	github.com/missdeer/golib v1.0.3
 	github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474 // indirect
 	github.com/mrexodia/wray v0.0.0-20160318003008-78a2c1f284ff // indirect
 	github.com/nelsonken/gomf v0.0.0-20180504123937-a9dd2f9deae9
-	github.com/nicksnyder/go-i18n v1.4.0 // indirect
-	github.com/onsi/ginkgo v1.6.0 // indirect
-	github.com/onsi/gomega v1.4.1 // indirect
 	github.com/paulrosania/go-charset v0.0.0-20190326053356-55c9d7a5834c
-	github.com/pborman/uuid v0.0.0-20160216163710-c55201b03606 // indirect
 	github.com/rs/xid v1.2.1
 	github.com/russross/blackfriday v1.5.2
 	github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca
 	github.com/shazow/ssh-chat v1.8.3-0.20200308224626-80ddf1f43a98
-	github.com/sirupsen/logrus v1.4.2
-	github.com/slack-go/slack v0.6.3-0.20200228121756-f56d616d5901
-	github.com/spf13/viper v1.6.1
-	github.com/stretchr/testify v1.4.0
-	github.com/technoweenie/multipartstreamer v1.0.1 // indirect
+	github.com/sirupsen/logrus v1.6.0
+	github.com/slack-go/slack v0.6.6
+	github.com/spf13/viper v1.7.1
+	github.com/stretchr/testify v1.6.1
+	github.com/writeas/go-strip-markdown v2.0.1+incompatible
 	github.com/x-cray/logrus-prefixed-formatter v0.5.2 // indirect
-	github.com/yaegashi/msgraph.go v0.1.2
-	github.com/zfjagann/golang-ring v0.0.0-20190106091943-a88bb6aef447
-	golang.org/x/image v0.0.0-20191214001246-9130b4cfad52
-	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
-	gopkg.in/fsnotify.v1 v1.4.7 // indirect
-	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
-	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
+	github.com/yaegashi/msgraph.go v0.1.4
+	github.com/zfjagann/golang-ring v0.0.0-20190304061218-d34796e0a6c2
+	golang.org/x/image v0.0.0-20200801110659-972c09e46d76
+	golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43
+	gomod.garykim.dev/nc-talk v0.1.3
+	gopkg.in/olahol/melody.v1 v1.0.0-20170518105555-d52139073376
 )
 
-//replace github.com/bwmarrin/discordgo v0.20.2 => github.com/matterbridge/discordgo v0.18.1-0.20200109173909-ed873362fa43
-
 go 1.13

matterbridge.go

@@ -4,6 +4,7 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"runtime"
 	"strings"
 
 	"github.com/42wim/matterbridge/bridge/config"
@@ -15,7 +16,7 @@ import (
 )
 
 var (
-	version = "1.17.1"
+	version = "1.18.3"
 	githash string
 
 	flagConfig  = flag.String("conf", "matterbridge.toml", "config file")
@@ -50,6 +51,15 @@ func main() {
 	cfg := config.NewConfig(rootLogger, *flagConfig)
 	cfg.BridgeValues().General.Debug = *flagDebug
 
+	// if logging to a file, ensure it is closed when the program terminates
+	// nolint:errcheck
+	defer func() {
+		if f, ok := rootLogger.Out.(*os.File); ok {
+			f.Sync()
+			f.Close()
+		}
+	}()
+
 	r, err := gateway.NewRouter(rootLogger, cfg, bridgemap.FullMap)
 	if err != nil {
 		logger.Fatalf("Starting gateway failed: %s", err)
@@ -67,17 +77,31 @@ func setupLogger() *logrus.Logger {
 		Formatter: &prefixed.TextFormatter{
 			PrefixPadding: 13,
 			DisableColors: true,
-			FullTimestamp: true,
 		},
 		Level: logrus.InfoLevel,
 	}
 	if *flagDebug || os.Getenv("DEBUG") == "1" {
+		logger.SetReportCaller(true)
 		logger.Formatter = &prefixed.TextFormatter{
-			PrefixPadding:   13,
-			DisableColors:   true,
-			FullTimestamp:   false,
-			ForceFormatting: true,
+			PrefixPadding: 13,
+			DisableColors: true,
+			FullTimestamp: false,
+
+			CallerFormatter: func(function, file string) string {
+				return fmt.Sprintf(" [%s:%s]", function, file)
+			},
+			CallerPrettyfier: func(f *runtime.Frame) (string, string) {
+				sp := strings.SplitAfter(f.File, "/matterbridge/")
+				filename := f.File
+				if len(sp) > 1 {
+					filename = sp[1]
+				}
+				s := strings.Split(f.Function, ".")
+				funcName := s[len(s)-1]
+				return funcName, fmt.Sprintf("%s:%d", filename, f.Line)
+			},
 		}
+
 		logger.Level = logrus.DebugLevel
 		logger.WithFields(logrus.Fields{"prefix": "main"}).Info("Enabling debug logging.")
 	}

matterbridge.toml.sample

@@ -103,6 +103,10 @@ ColorNicks=false
 #OPTIONAL (default empty)
 RunCommands=["PRIVMSG user hello","PRIVMSG chanserv something"]
 
+#StripMarkdown strips markdown from messages
+#OPTIONAL (default false)
+StripMarkdown=false
+
 #Nicks you want to ignore.
 #Regular expressions supported
 #Messages from those users will not be sent to other bridges.
@@ -177,6 +181,12 @@ StripNick=false
 #OPTIONAL (default false)
 ShowTopicChange=false
 
+#Delay in milliseconds between channel joins
+#Only useful when you have a LOT of channels to join
+#See https://github.com/42wim/matterbridge/issues/1084
+#OPTIONAL (default 0)
+JoinDelay=0
+
 ###################################################################
 #XMPP section
 ###################################################################
@@ -1203,6 +1213,11 @@ Password="yourpass"
 #OPTIONAL (default false)
 NoHomeServerSuffix=false
 
+#Whether to disable sending of HTML content to matrix
+#See https://github.com/42wim/matterbridge/issues/1022
+#OPTIONAL (default false)
+HTMLDisable=false
+
 ## RELOADABLE SETTINGS
 ## Settings below can be reloaded by editing the file
 
@@ -1368,7 +1383,27 @@ StripNick=false
 #OPTIONAL (default false)
 ShowTopicChange=false
 
+###################################################################
+#
+# NCTalk (Nextcloud Talk)
+#
+###################################################################
 
+[nctalk.bridge]
+
+# Url of your Nextcloud server
+Server = "https://cloud.youdomain.me"
+
+# Enable to not verify the certificate on your Nextcloud server.
+# e.g. when using selfsigned certificates
+# OPTIONAL (default false)
+SkipTLSVerify=true
+
+# Username of the bot
+Login = "talkuser"
+
+# Password of the bot
+Password = "talkuserpass"
 
 ###################################################################
 #
@@ -1510,6 +1545,8 @@ Buffer=1000
 
 #Bearer token used for authentication
 #curl -H "Authorization: Bearer token" http://localhost:4242/api/messages
+# https://github.com/vi/websocat
+# websocat -H="Authorization: Bearer token" ws://127.0.0.1:4242/api/websocket
 #OPTIONAL (no authorization if token is empty)
 Token="mytoken"
 
@@ -1589,6 +1626,14 @@ MediaDownloadBlacklist=[".html$",".htm$"]
 #OPTIONAL (default false)
 IgnoreFailureOnStart=false
 
+#LogFile defines the location of a file to write logs into, rather
+#than stdout.
+#Logging will still happen on stdout if the file cannot be open for
+#writing, or if the value is empty. Note that the log won't roll, so
+#you might want to use logrotate(8) with this feature.
+#OPTIONAL (default empty)
+LogFile="/var/log/matterbridge.log"
+
 ###################################################################
 #Tengo configuration
 ###################################################################
@@ -1679,34 +1724,46 @@ enable=true
     # REQUIRED
     account="irc.freenode"
 
-    # channel to connect on that account
-    # How to specify them for the different bridges:
+    # The channel key in each gateway is mapped to a similar group chat ID on the chat platform
+    # To find the group chat ID for different platforms, refer to the table below
     #
-    # irc        - #channel (# is required) (this needs to be lowercase!)
-    # mattermost - channel (the channel name as seen in the URL, not the displayname)
-    # gitter     - username/room
-    # xmpp       - channel
-    # slack      - channel (without the #)
-    #            - ID:C123456 (where C123456 is the channel ID) does not work with webhook
-    # discord    - channel (without the #)
-    #            - ID:123456789 (where 123456789 is the channel ID)
-    #               (https://github.com/42wim/matterbridge/issues/57)
-    #            - category/channel (without the #) if you're using discord categories to group your channels
-    # telegram   - chatid (a large negative number, eg -123456789)
-    #             see (https://www.linkedin.com/pulse/telegram-bots-beginners-marco-frau)
-    # hipchat    - id_channel (see https://www.hipchat.com/account/xmpp for the correct channel)
-    # rocketchat - #channel (# is required (also needed for private channels!)
-    # matrix     - #channel:server (eg #yourchannel:matrix.org)
-    #            - encrypted rooms are not supported in matrix
-    # msteams    - 19:xxxxxxxxxxxxxxxxxxxxxxxxxx@thread.skype
-    #            - You'll find the channel ID in the URL in the threadId=19:82abcxxxxxxxxx@thread.skype
-    # steam      - chatid (a large number).
-    #             The number in the URL when you click "enter chat room" in the browser
-    # whatsapp   - 48111222333-123455678999@g.us A unique group JID;
-    #              if you specify an empty string bridge will list all the possibilities
-    #            - "Group Name" if you specify a group name the bridge will hint its JID to specify
-    #              as group names might change in time and contain weird emoticons
-    # zulip      - stream/topic:topicname (without the #)
+    # Platform   |   Identifier name  |            Example            | Description
+    # -------------------------------------------------------------------------------------------------------------------------------------
+    #            |      channel       |            general            | Do not include the # symbol
+    #  discord   |    channel id      |          ID:123456789         | See https://github.com/42wim/matterbridge/issues/57
+    #            | category/channel   |          Media/gaming         | Without # symbol. If you're using discord categories to group your channels
+    # -------------------------------------------------------------------------------------------------------------------------------------
+    #   gitter   |  username/room     |            general            | As seen in the gitter.im URL
+    # -------------------------------------------------------------------------------------------------------------------------------------
+    #   hipchat  |    id_channel      |         example needed        | See https://www.hipchat.com/account/xmpp for the correct channel
+    # -------------------------------------------------------------------------------------------------------------------------------------
+    #    irc     |      channel       |            #general           | The # symbol is required and should be lowercase!
+    # -------------------------------------------------------------------------------------------------------------------------------------
+    # mattermost |      channel       |            general            | This is the channel name as seen in the URL, not the display name
+    # -------------------------------------------------------------------------------------------------------------------------------------
+    #   matrix   | #channel:server    |    #yourchannel:matrix.org    | Encrypted rooms are not supported in matrix
+    # -------------------------------------------------------------------------------------------------------------------------------------
+    #   msteams  |      threadId      |    19:82abcxx@thread.skype    | You'll find the threadId in the URL
+    # -------------------------------------------------------------------------------------------------------------------------------------
+    # rocketchat |      channel       |            #channel           | # is required for private channels too
+    # -------------------------------------------------------------------------------------------------------------------------------------
+    #   slack    |   channel name     |            general            | Do not include the # symbol
+    #            |    channel id      |           ID:C123456          | The underlying ID of a channel. This doesn't work with
+    # -------------------------------------------------------------------------------------------------------------------------------------
+    #   steam    |      chatid        |         example needed        | The number in the URL when you click "enter chat room" in the browser
+    # -------------------------------------------------------------------------------------------------------------------------------------
+    #   nctalk   |      token         |           xs25tz5y            | The token in the URL when you are in a chat. It will be the last part of the URL.
+    # -------------------------------------------------------------------------------------------------------------------------------------
+    #  telegram  |      chatid        |          -123456789           | A large negative number. see https://www.linkedin.com/pulse/telegram-bots-beginners-marco-frau
+    # -------------------------------------------------------------------------------------------------------------------------------------
+    #  whatsapp  |     group JID      | 48111222333-123455678999@g.us | A unique group JID. If you specify an empty string, bridge will list all the possibilities
+    #            |    "Group Name"    |         "Family Chat"         | if you specify a group name, the bridge will find hint the JID to specify. Names can change over time and are not stable.
+    # -------------------------------------------------------------------------------------------------------------------------------------
+    #    xmpp    |      channel       |            general            | The room name
+    # -------------------------------------------------------------------------------------------------------------------------------------
+    #   zulip    | stream/topic:topic |     general/off-topic:food    | Do not use the # when specifying a topic
+    # -------------------------------------------------------------------------------------------------------------------------------------
+
     #
     # REQUIRED
     channel="#testing"

matterclient/channels.go

@@ -4,7 +4,7 @@ import (
 	"errors"
 	"strings"
 
-	"github.com/mattermost/mattermost-server/model"
+	"github.com/mattermost/mattermost-server/v5/model"
 )
 
 // GetChannels returns all channels we're members off
@@ -167,7 +167,7 @@ func (m *MMClient) JoinChannel(channelId string) error { //nolint:golint
 }
 
 func (m *MMClient) UpdateChannelsTeam(teamID string) error {
-	mmchannels, resp := m.Client.GetChannelsForTeamForUser(teamID, m.User.Id, "")
+	mmchannels, resp := m.Client.GetChannelsForTeamForUser(teamID, m.User.Id, false, "")
 	if resp.Error != nil {
 		return errors.New(resp.Error.DetailedError)
 	}

matterclient/helpers.go

@@ -13,7 +13,7 @@ import (
 
 	"github.com/gorilla/websocket"
 	"github.com/jpillora/backoff"
-	"github.com/mattermost/mattermost-server/model"
+	"github.com/mattermost/mattermost-server/v5/model"
 )
 
 func (m *MMClient) doLogin(firstConnection bool, b *backoff.Backoff) error {
@@ -154,7 +154,7 @@ func (m *MMClient) initUser() error {
 
 		t := &Team{Team: team, Users: usermap, Id: team.Id}
 
-		mmchannels, resp := m.Client.GetChannelsForTeamForUser(team.Id, m.User.Id, "")
+		mmchannels, resp := m.Client.GetChannelsForTeamForUser(team.Id, m.User.Id, false, "")
 		if resp.Error != nil {
 			return resp.Error
 		}

matterclient/matterclient.go

@@ -11,7 +11,7 @@ import (
 	lru "github.com/hashicorp/golang-lru"
 	"github.com/jpillora/backoff"
 	prefixed "github.com/matterbridge/logrus-prefixed-formatter"
-	"github.com/mattermost/mattermost-server/model"
+	"github.com/mattermost/mattermost-server/v5/model"
 	"github.com/sirupsen/logrus"
 )
 
@@ -69,6 +69,7 @@ type MMClient struct {
 	logger     *logrus.Entry
 	rootLogger *logrus.Logger
 	lruCache   *lru.Cache
+	allevents  bool
 }
 
 // New will instantiate a new Matterclient with the specified login details without connecting.
@@ -119,6 +120,10 @@ func (m *MMClient) SetLogLevel(level string) {
 	}
 }
 
+func (m *MMClient) EnableAllEvents() {
+	m.allevents = true
+}
+
 // Login tries to connect the client with the loging details with which it was initialized.
 func (m *MMClient) Login() error {
 	// check if this is a first connect or a reconnection
@@ -220,6 +225,10 @@ func (m *MMClient) WsReceiver() {
 					continue
 				}
 			}
+			if m.allevents {
+				m.MessageChan <- msg
+				continue
+			}
 			switch msg.Raw.Event {
 			case model.WEBSOCKET_EVENT_USER_ADDED,
 				model.WEBSOCKET_EVENT_USER_REMOVED,

matterclient/messages.go

@@ -3,7 +3,7 @@ package matterclient
 import (
 	"strings"
 
-	"github.com/mattermost/mattermost-server/model"
+	"github.com/mattermost/mattermost-server/v5/model"
 )
 
 func (m *MMClient) parseActionPost(rmsg *Message) {

matterclient/users.go

@@ -4,7 +4,7 @@ import (
 	"errors"
 	"time"
 
-	"github.com/mattermost/mattermost-server/model"
+	"github.com/mattermost/mattermost-server/v5/model"
 )
 
 func (m *MMClient) GetNickName(userId string) string { //nolint:golint

tgs.Dockerfile

@@ -0,0 +1,38 @@
+FROM alpine AS builder
+
+COPY . /go/src/github.com/42wim/matterbridge
+RUN apk add \
+    go \
+    git \
+    gcc \
+    musl-dev \
+  && cd /go/src/github.com/42wim/matterbridge \
+  && export GOPATH=/go \
+  && go get \
+  && go build -x -ldflags "-X main.githash=$(git log --pretty=format:'%h' -n 1)" -o /bin/matterbridge
+
+FROM alpine
+RUN apk --no-cache add \
+    ca-certificates \
+    cairo \
+    libjpeg-turbo \
+    mailcap \
+    py3-webencodings \
+    python3 \
+  && apk --no-cache add --virtual .compile \
+    gcc \
+    libffi-dev \
+    libjpeg-turbo-dev \
+    musl-dev \
+    py3-pip \
+    py3-wheel \
+    python3-dev \
+    zlib-dev \
+  && pip3 install --no-cache-dir lottie[PNG] \
+  && apk --no-cache del .compile
+
+COPY --from=builder /bin/matterbridge /bin/matterbridge
+RUN mkdir /etc/matterbridge \
+  && touch /etc/matterbridge/matterbridge.toml \
+  && ln -sf /matterbridge.toml /etc/matterbridge/matterbridge.toml
+ENTRYPOINT ["/bin/matterbridge", "-conf", "/etc/matterbridge/matterbridge.toml"]

vendor/github.com/Rhymen/go-whatsapp/README.md

@@ -70,6 +70,10 @@ func (myHandler) HandleContactMessage(message whatsapp.ContactMessage) {
 	fmt.Println(message)
 }
 
+func (myHandler) HandleBatteryMessage(msg whatsapp.BatteryMessage) {
+	fmt.Println(message)
+}
+
 wac.AddHandler(myHandler{})
 ```
 The message handlers are all optional, you don't need to implement anything but the error handler to implement the interface. The ImageMessage, VideoMessage, AudioMessage and DocumentMessage provide a Download function to get the media data.

vendor/github.com/Rhymen/go-whatsapp/binary/proto/def.proto

@@ -56,6 +56,8 @@ message Location {
 }
 
 message Point {
+    optional int32 xDeprecated = 1;
+    optional int32 yDeprecated = 2;
     optional double x = 3;
     optional double y = 4;
 }
@@ -93,6 +95,7 @@ message ContextInfo {
     optional AdReplyInfo quotedAd = 23;
     optional MessageKey placeholderKey = 24;
     optional uint32 expiration = 25;
+    optional int64 ephemeralSettingTimestamp = 26;
 }
 
 message SenderKeyDistributionMessage {
@@ -136,6 +139,11 @@ message LocationMessage {
     optional string name = 3;
     optional string address = 4;
     optional string url = 5;
+    optional bool isLive = 6;
+    optional uint32 accuracyInMeters = 7;
+    optional float speedInMps = 8;
+    optional uint32 degreesClockwiseFromMagneticNorth = 9;
+    optional string comment = 11;
     optional bytes jpegThumbnail = 16;
     optional ContextInfo contextInfo = 17;
 }
@@ -238,9 +246,29 @@ message ProtocolMessage {
     enum PROTOCOL_MESSAGE_TYPE {
         REVOKE = 0;
         EPHEMERAL_SETTING = 3;
+        EPHEMERAL_SYNC_RESPONSE = 4;
+        HISTORY_SYNC_NOTIFICATION = 5;
     }
     optional PROTOCOL_MESSAGE_TYPE type = 2;
     optional uint32 ephemeralExpiration = 4;
+    optional int64 ephemeralSettingTimestamp = 5;
+    optional HistorySyncNotification historySyncNotification = 6;
+}
+
+message HistorySyncNotification {
+    optional bytes fileSha256 = 1;
+    optional uint64 fileLength = 2;
+    optional bytes mediaKey = 3;
+    optional bytes fileEncSha256 = 4;
+    optional string directPath = 5;
+    enum HISTORY_SYNC_NOTIFICATION_HISTORYSYNCTYPE {
+        INITIAL_BOOTSTRAP = 0;
+        INITIAL_STATUS_V3 = 1;
+        FULL = 2;
+        RECENT = 3;
+    }
+    optional HISTORY_SYNC_NOTIFICATION_HISTORYSYNCTYPE syncType = 6;
+    optional uint32 chunkOrder = 7;
 }
 
 message ContactsArrayMessage {
@@ -355,6 +383,8 @@ message StickerMessage {
     optional int64 mediaKeyTimestamp = 10;
     optional uint32 firstFrameLength = 11;
     optional bytes firstFrameSidecar = 12;
+    optional bool isAnimated = 13;
+    optional bytes pngThumbnail = 16;
     optional ContextInfo contextInfo = 17;
 }
 
@@ -401,6 +431,12 @@ message TemplateButtonReplyMessage {
     optional uint32 selectedIndex = 4;
 }
 
+message CatalogSnapshot {
+    optional ImageMessage catalogImage = 1;
+    optional string title = 2;
+    optional string description = 3;
+}
+
 message ProductSnapshot {
     optional ImageMessage productImage = 1;
     optional string productId = 2;
@@ -417,6 +453,7 @@ message ProductSnapshot {
 message ProductMessage {
     optional ProductSnapshot product = 1;
     optional string businessOwnerJid = 2;
+    optional CatalogSnapshot catalog = 4;
     optional ContextInfo contextInfo = 17;
 }
 
@@ -513,6 +550,8 @@ message WebFeatures {
     optional WEB_FEATURES_FLAG templateMessage = 30;
     optional WEB_FEATURES_FLAG templateMessageInteractivity = 31;
     optional WEB_FEATURES_FLAG ephemeralMessages = 32;
+    optional WEB_FEATURES_FLAG e2ENotificationSync = 33;
+    optional WEB_FEATURES_FLAG recentStickersV2 = 34;
 }
 
 message TabletNotificationsInfo {
@@ -537,6 +576,11 @@ message WebNotificationsInfo {
 }
 
 message PaymentInfo {
+    enum PAYMENT_INFO_CURRENCY {
+        UNKNOWN_CURRENCY = 0;
+        INR = 1;
+    }
+    optional PAYMENT_INFO_CURRENCY currencyDeprecated = 1;
     optional uint64 amount1000 = 2;
     optional string receiverJid = 3;
     enum PAYMENT_INFO_STATUS {
@@ -559,6 +603,37 @@ message PaymentInfo {
     optional uint64 expiryTimestamp = 7;
     optional bool futureproofed = 8;
     optional string currency = 9;
+    enum PAYMENT_INFO_TXNSTATUS {
+        UNKNOWN = 0;
+        PENDING_SETUP = 1;
+        PENDING_RECEIVER_SETUP = 2;
+        INIT = 3;
+        SUCCESS = 4;
+        COMPLETED = 5;
+        FAILED = 6;
+        FAILED_RISK = 7;
+        FAILED_PROCESSING = 8;
+        FAILED_RECEIVER_PROCESSING = 9;
+        FAILED_DA = 10;
+        FAILED_DA_FINAL = 11;
+        REFUNDED_TXN = 12;
+        REFUND_FAILED = 13;
+        REFUND_FAILED_PROCESSING = 14;
+        REFUND_FAILED_DA = 15;
+        EXPIRED_TXN = 16;
+        AUTH_CANCELED = 17;
+        AUTH_CANCEL_FAILED_PROCESSING = 18;
+        AUTH_CANCEL_FAILED = 19;
+        COLLECT_INIT = 20;
+        COLLECT_SUCCESS = 21;
+        COLLECT_FAILED = 22;
+        COLLECT_FAILED_RISK = 23;
+        COLLECT_REJECTED = 24;
+        COLLECT_EXPIRED = 25;
+        COLLECT_CANCELED = 26;
+        COLLECT_CANCELLING = 27;
+    }
+    optional PAYMENT_INFO_TXNSTATUS txnStatus = 10;
 }
 
 message WebMessageInfo {
@@ -668,4 +743,5 @@ message WebMessageInfo {
     optional PaymentInfo quotedPaymentInfo = 31;
     optional uint64 ephemeralStartTimestamp = 32;
     optional uint32 ephemeralDuration = 33;
-}
+}
+

vendor/github.com/Rhymen/go-whatsapp/conn.go

@@ -88,8 +88,11 @@ type Conn struct {
 	Store          *Store
 	ServerLastSeen time.Time
 
+	timeTag string // last 3 digits obtained after a successful login takeover
+
 	longClientName  string
 	shortClientName string
+	clientVersion   string
 
 	loginSessionLock sync.RWMutex
 	Proxy            func(*http.Request) (*url.URL, error)
@@ -121,6 +124,7 @@ func NewConn(timeout time.Duration) (*Conn, error) {
 
 		longClientName:  "github.com/rhymen/go-whatsapp",
 		shortClientName: "go-whatsapp",
+		clientVersion:   "0.1.0",
 	}
 	return wac, wac.connect()
 }
@@ -135,6 +139,7 @@ func NewConnWithProxy(timeout time.Duration, proxy func(*http.Request) (*url.URL
 
 		longClientName:  "github.com/rhymen/go-whatsapp",
 		shortClientName: "go-whatsapp",
+		clientVersion:   "0.1.0",
 		Proxy:           proxy,
 	}
 	return wac, wac.connect()
@@ -153,8 +158,8 @@ func (wac *Conn) connect() (err error) {
 	}()
 
 	dialer := &websocket.Dialer{
-		ReadBufferSize:   25 * 1024 * 1024,
-		WriteBufferSize:  10 * 1024 * 1024,
+		ReadBufferSize:   0,
+		WriteBufferSize:  0,
 		HandshakeTimeout: wac.msgTimeout,
 		Proxy:            wac.Proxy,
 	}
@@ -243,3 +248,11 @@ func (wac *Conn) keepAlive(minIntervalMs int, maxIntervalMs int) {
 		}
 	}
 }
+
+func (wac *Conn) GetConnected() bool {
+	return  wac.connected
+}
+
+func (wac *Conn) GetLoggedIn() bool {
+	return  wac.loggedIn
+}

vendor/github.com/Rhymen/go-whatsapp/errors.go

@@ -2,6 +2,7 @@ package whatsapp
 
 import (
 	"fmt"
+
 	"github.com/pkg/errors"
 )
 
@@ -20,6 +21,7 @@ var (
 	ErrServerRespondedWith404     = errors.New("server responded with status 404")
 	ErrMediaDownloadFailedWith404 = errors.New("download failed with status code 404")
 	ErrMediaDownloadFailedWith410 = errors.New("download failed with status code 410")
+	ErrInvalidWebsocket           = errors.New("invalid websocket")
 )
 
 type ErrConnectionFailed struct {

vendor/github.com/Rhymen/go-whatsapp/go.mod

@@ -6,7 +6,7 @@ require (
 	github.com/Rhymen/go-whatsapp/examples/sendImage v0.0.0-20190325075644-cc2581bbf24d // indirect
 	github.com/Rhymen/go-whatsapp/examples/sendTextMessages v0.0.0-20190325075644-cc2581bbf24d // indirect
 	github.com/golang/protobuf v1.3.0
-	github.com/gorilla/websocket v1.4.0
+	github.com/gorilla/websocket v1.4.1
 	github.com/pkg/errors v0.8.1
 	golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2
 )

vendor/github.com/Rhymen/go-whatsapp/go.sum

@@ -12,8 +12,9 @@ github.com/Rhymen/go-whatsapp/examples/sendTextMessages v0.0.0-20190325075644-cc
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.0 h1:kbxbvI4Un1LUWKxufD+BiE6AEExYYgkQLQmLFqA1LFk=
 github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0=
-github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM=
+github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg=
 github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
 github.com/mattn/go-isatty v0.0.5 h1:tHXDdz1cpzGaovsTB+TVB8q90WEokoVmfMqoVcrLUgw=

vendor/github.com/Rhymen/go-whatsapp/handler.go

@@ -133,6 +133,14 @@ type ChatListHandler interface {
 	HandleChatList(contacts []Chat)
 }
 
+/**
+The BatteryMessageHandler interface needs to be implemented to receive percentage the device connected dispatched by the dispatcher.
+*/
+type BatteryMessageHandler interface {
+	Handler
+	HandleBatteryMessage(battery BatteryMessage)
+}
+
 /*
 AddHandler adds an handler to the list of handler that receive dispatched messages.
 The provided handler must at least implement the Handler interface. Additionally implemented
@@ -285,6 +293,17 @@ func (wac *Conn) handleWithCustomHandlers(message interface{}, handlers []Handle
 				}
 			}
 		}
+	
+	case BatteryMessage:
+		for _, h := range handlers {
+			if x, ok := h.(BatteryMessageHandler); ok {
+				if wac.shouldCallSynchronously(h) {
+					x.HandleBatteryMessage(m)
+				} else {
+					go x.HandleBatteryMessage(m)
+				}
+			}
+		}
 
 	case *proto.WebMessageInfo:
 		for _, h := range handlers {
@@ -379,6 +398,10 @@ func (wac *Conn) dispatch(msg interface{}) {
 						wac.handle(ParseProtoMessage(v))
 					}
 				}
+			} else if con, ok := message.Content.([]binary.Node); ok {
+				for a := range con {
+					wac.handle(ParseNodeMessage(con[a]))
+				}
 			}
 		} else if message.Description == "response" && message.Attributes["type"] == "contacts" {
 			wac.updateContacts(message.Content)

vendor/github.com/Rhymen/go-whatsapp/media.go

@@ -10,10 +10,8 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"mime/multipart"
 	"net/http"
-	"os"
-	"strings"
+	"net/url"
 	"time"
 
 	"github.com/Rhymen/go-whatsapp/crypto/cbc"
@@ -95,7 +93,53 @@ func downloadMedia(url string) (file []byte, mac []byte, err error) {
 	return data[:n-10], data[n-10 : n], nil
 }
 
-func (wac *Conn) Upload(reader io.Reader, appInfo MediaType) (url string, mediaKey []byte, fileEncSha256 []byte, fileSha256 []byte, fileLength uint64, err error) {
+                                                                                
+type MediaConn struct {                                                         
+        Status int `json:"status"`                                              
+        MediaConn struct {                                                      
+                Auth string `json:"auth"`                                       
+                TTL int `json:"ttl"`                                            
+                Hosts []struct {                                                
+                        Hostname string `json:"hostname"`                       
+                        IPs []interface{}  `json:"ips"`                                              
+                } `json:"hosts"`                                                
+        } `json:"media_conn"`                                                   
+}
+
+
+
+func (wac *Conn) queryMediaConn() (hostname, auth string, ttl int, err error) {
+	queryReq := []interface{}{"query", "mediaConn"}
+	ch, err := wac.writeJson(queryReq)
+	if err != nil {
+		return "", "", 0, err
+	}
+
+	var resp MediaConn
+	select {
+	case r := <-ch:
+		if err = json.Unmarshal([]byte(r), &resp); err != nil {
+			return "", "", 0, fmt.Errorf("error decoding query media conn response: %v", err)
+		}
+	case <-time.After(wac.msgTimeout):
+		return "", "", 0, fmt.Errorf("query media conn timed out")
+	}
+
+	if resp.Status != 200 {
+		return "", "", 0, fmt.Errorf("query media conn responded with %d", resp.Status)
+	}
+
+	return resp.MediaConn.Hosts[0].Hostname, resp.MediaConn.Auth, resp.MediaConn.TTL, nil
+}
+
+var mediaTypeMap = map[MediaType]string{
+	MediaImage: "/mms/image",
+	MediaVideo: "/mms/video",
+	MediaDocument: "/mms/document",
+	MediaAudio: "/mms/audio",
+}
+
+func (wac *Conn) Upload(reader io.Reader, appInfo MediaType) (downloadURL string, mediaKey []byte, fileEncSha256 []byte, fileSha256 []byte, fileLength uint64, err error) {
 	data, err := ioutil.ReadAll(reader)
 	if err != nil {
 		return "", nil, nil, nil, 0, err
@@ -128,67 +172,30 @@ func (wac *Conn) Upload(reader io.Reader, appInfo MediaType) (url string, mediaK
 	sha.Write(append(enc, mac...))
 	fileEncSha256 = sha.Sum(nil)
 
-	var filetype string
-	switch appInfo {
-	case MediaImage:
-		filetype = "image"
-	case MediaAudio:
-		filetype = "audio"
-	case MediaDocument:
-		filetype = "document"
-	case MediaVideo:
-		filetype = "video"
+	hostname, auth, _, err := wac.queryMediaConn()
+	token := base64.URLEncoding.EncodeToString(fileEncSha256)
+	q := url.Values{
+		"auth":  []string{auth},
+		"token": []string{token},
+	}
+	path := mediaTypeMap[appInfo]
+	uploadURL := url.URL{
+		Scheme:   "https",
+		Host:     hostname,
+		Path:     fmt.Sprintf("%s/%s", path, token),
+		RawQuery: q.Encode(),
 	}
 
-	uploadReq := []interface{}{"action", "encr_upload", filetype, base64.StdEncoding.EncodeToString(fileEncSha256)}
-	ch, err := wac.writeJson(uploadReq)
+	body := bytes.NewReader(append(enc, mac...))
+
+	req, err := http.NewRequest("POST", uploadURL.String(), body)
 	if err != nil {
 		return "", nil, nil, nil, 0, err
 	}
 
-	var resp map[string]interface{}
-	select {
-	case r := <-ch:
-		if err = json.Unmarshal([]byte(r), &resp); err != nil {
-			return "", nil, nil, nil, 0, fmt.Errorf("error decoding upload response: %v", err)
-		}
-	case <-time.After(wac.msgTimeout):
-		return "", nil, nil, nil, 0, fmt.Errorf("restore session init timed out")
-	}
-
-	if int(resp["status"].(float64)) != 200 {
-		return "", nil, nil, nil, 0, fmt.Errorf("upload responsed with %d", resp["status"])
-	}
-
-	var b bytes.Buffer
-	w := multipart.NewWriter(&b)
-	hashWriter, err := w.CreateFormField("hash")
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "%v\n", err)
-	}
-	io.Copy(hashWriter, strings.NewReader(base64.StdEncoding.EncodeToString(fileEncSha256)))
-
-	fileWriter, err := w.CreateFormFile("file", "blob")
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "%v\n", err)
-	}
-	io.Copy(fileWriter, bytes.NewReader(append(enc, mac...)))
-	err = w.Close()
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "%v\n", err)
-	}
-
-	req, err := http.NewRequest("POST", resp["url"].(string), &b)
-	if err != nil {
-		return "", nil, nil, nil, 0, err
-	}
-
-	req.Header.Set("Content-Type", w.FormDataContentType())
 	req.Header.Set("Origin", "https://web.whatsapp.com")
 	req.Header.Set("Referer", "https://web.whatsapp.com/")
 
-	req.URL.Query().Set("f", "j")
-
 	client := &http.Client{}
 	// Submit the request
 	res, err := client.Do(req)

vendor/github.com/Rhymen/go-whatsapp/message.go

@@ -81,7 +81,7 @@ func (wac *Conn) Send(msg interface{}) (string, error) {
 			return "ERROR", fmt.Errorf("error decoding sending response: %v\n", err)
 		}
 		if int(resp["status"].(float64)) != 200 {
-			return "ERROR", fmt.Errorf("message sending responded with %d", resp["status"])
+			return "ERROR", fmt.Errorf("message sending responded with %v", resp["status"])
 		}
 		if int(resp["status"].(float64)) == 200 {
 			return getMessageInfo(msgProto).Id, nil
@@ -105,6 +105,105 @@ func (wac *Conn) sendProto(p *proto.WebMessageInfo) (<-chan string, error) {
 	return wac.writeBinary(n, message, ignore, p.Key.GetId())
 }
 
+// RevokeMessage revokes a message (marks as "message removed") for everyone
+func (wac *Conn) RevokeMessage(remotejid, msgid string, fromme bool) (revokeid string, err error) {
+	// create a revocation ID (required)
+	rawrevocationID := make([]byte, 10)
+	rand.Read(rawrevocationID)
+	revocationID := strings.ToUpper(hex.EncodeToString(rawrevocationID))
+	//
+	ts := uint64(time.Now().Unix())
+	status := proto.WebMessageInfo_PENDING
+	mtype := proto.ProtocolMessage_REVOKE
+
+	revoker := &proto.WebMessageInfo{
+		Key: &proto.MessageKey{
+			FromMe:    &fromme,
+			Id:        &revocationID,
+			RemoteJid: &remotejid,
+		},
+		MessageTimestamp: &ts,
+		Message: &proto.Message{
+			ProtocolMessage: &proto.ProtocolMessage{
+				Type: &mtype,
+				Key: &proto.MessageKey{
+					FromMe:    &fromme,
+					Id:        &msgid,
+					RemoteJid: &remotejid,
+				},
+			},
+		},
+		Status: &status,
+	}
+	if _, err := wac.Send(revoker); err != nil {
+		return revocationID, err
+	}
+	return revocationID, nil
+}
+
+// DeleteMessage deletes a single message for the user (removes the msgbox). To
+// delete the message for everyone, use RevokeMessage
+func (wac *Conn) DeleteMessage(remotejid, msgid string, fromMe bool) error {
+	ch, err := wac.deleteChatProto(remotejid, msgid, fromMe)
+	if err != nil {
+		return fmt.Errorf("could not send proto: %v", err)
+	}
+
+	select {
+	case response := <-ch:
+		var resp map[string]interface{}
+		if err = json.Unmarshal([]byte(response), &resp); err != nil {
+			return fmt.Errorf("error decoding deletion response: %v", err)
+		}
+		if int(resp["status"].(float64)) != 200 {
+			return fmt.Errorf("message deletion responded with %v", resp["status"])
+		}
+		if int(resp["status"].(float64)) == 200 {
+			return nil
+		}
+	case <-time.After(wac.msgTimeout):
+		return fmt.Errorf("deleting message timed out")
+	}
+
+	return nil
+}
+
+func (wac *Conn) deleteChatProto(remotejid, msgid string, fromMe bool) (<-chan string, error) {
+	tag := fmt.Sprintf("%s.--%d", wac.timeTag, wac.msgCount)
+
+	owner := "true"
+	if !fromMe {
+		owner = "false"
+	}
+	n := binary.Node{
+		Description: "action",
+		Attributes: map[string]string{
+			"epoch": strconv.Itoa(wac.msgCount),
+			"type":  "set",
+		},
+		Content: []interface{}{
+			binary.Node{
+				Description: "chat",
+				Attributes: map[string]string{
+					"type":  "clear",
+					"jid":   remotejid,
+					"media": "true",
+				},
+				Content: []binary.Node{
+					{
+						Description: "item",
+						Attributes: map[string]string{
+							"owner": owner,
+							"index": msgid,
+						},
+					},
+				},
+			},
+		},
+	}
+	return wac.writeBinary(n, chat, expires|skipOffline, tag)
+}
+
 func init() {
 	rand.Seed(time.Now().UTC().UnixNano())
 }
@@ -744,3 +843,38 @@ func ParseProtoMessage(msg *proto.WebMessageInfo) interface{} {
 
 	return nil
 }
+
+
+/*
+BatteryMessage represents a battery level and charging state.
+*/
+type BatteryMessage struct {
+	Plugged bool
+	Powersave bool
+	Percentage int
+}
+
+func getBatteryMessage(msg map[string]string) BatteryMessage {
+	plugged, _ := strconv.ParseBool(msg["live"])
+	powersave, _ := strconv.ParseBool(msg["powersave"])
+	percentage, _ := strconv.Atoi(msg["value"])
+	batteryMessage := BatteryMessage{
+		Plugged: plugged,
+		Powersave: powersave,
+		Percentage: percentage,
+	}
+
+	return batteryMessage
+}
+
+
+func ParseNodeMessage(msg binary.Node) interface{} {
+	switch msg.Description {
+	case "battery":
+		return getBatteryMessage(msg.Attributes)
+	default:
+		//cannot match message
+	}
+
+	return nil
+}

vendor/github.com/Rhymen/go-whatsapp/read.go

@@ -5,17 +5,21 @@ import (
 	"crypto/sha256"
 	"encoding/json"
 	"fmt"
+	"io"
+	"io/ioutil"
+	"strings"
+
 	"github.com/Rhymen/go-whatsapp/binary"
 	"github.com/Rhymen/go-whatsapp/crypto/cbc"
 	"github.com/gorilla/websocket"
 	"github.com/pkg/errors"
-	"io"
-	"io/ioutil"
-	"strings"
 )
 
 func (wac *Conn) readPump() {
-	defer wac.wg.Done()
+	defer func() {
+		wac.wg.Done()
+		_, _ = wac.Disconnect()
+	}()
 
 	var readErr error
 	var msgType int
@@ -24,14 +28,15 @@ func (wac *Conn) readPump() {
 	for {
 		readerFound := make(chan struct{})
 		go func() {
-			msgType, reader, readErr = wac.ws.conn.NextReader()
+			if wac.ws != nil {
+				msgType, reader, readErr = wac.ws.conn.NextReader()
+			}
 			close(readerFound)
 		}()
 		select {
 		case <-readerFound:
 			if readErr != nil {
 				wac.handle(&ErrConnectionFailed{Err: readErr})
-				_, _ = wac.Disconnect()
 				return
 			}
 			msg, err := ioutil.ReadAll(reader)

vendor/github.com/Rhymen/go-whatsapp/session.go

@@ -18,7 +18,7 @@ import (
 )
 
 //represents the WhatsAppWeb client version
-var waVersion = []int{0, 3, 3324}
+var waVersion = []int{2, 2033, 7}
 
 /*
 Session contains session individual information. To be able to resume the connection without scanning the qr code
@@ -107,10 +107,10 @@ func CheckCurrentServerVersion() ([]int, error) {
 	}
 
 	b64ClientId := base64.StdEncoding.EncodeToString(clientId)
-	login := []interface{}{"admin", "init", waVersion, []string{wac.longClientName, wac.shortClientName}, b64ClientId, true}
+	login := []interface{}{"admin", "init", waVersion, []string{wac.longClientName, wac.shortClientName, wac.clientVersion}, b64ClientId, true}
 	loginChan, err := wac.writeJson(login)
 	if err != nil {
-		return nil, fmt.Errorf("error writing login", err)
+		return nil, fmt.Errorf("error writing login: %s", err.Error())
 	}
 
 	// Retrieve an answer from the websocket
@@ -123,7 +123,7 @@ func CheckCurrentServerVersion() ([]int, error) {
 
 	var resp map[string]interface{}
 	if err = json.Unmarshal([]byte(r), &resp); err != nil {
-		return nil, fmt.Errorf("error decoding login", err)
+		return nil, fmt.Errorf("error decoding login: %s", err.Error())
 	}
 
 	// Take the curr property as X.Y.Z and split it into as int slice
@@ -151,7 +151,7 @@ func (wac *Conn) SetClientName(long, short string) error {
 
 /*
 SetClientVersion sets WhatsApp client version
-Default value is 0.3.3324
+Default value is 0.4.2080
 */
 func (wac *Conn) SetClientVersion(major int, minor int, patch int) {
 	waVersion = []int{major, minor, patch}
@@ -213,7 +213,7 @@ func (wac *Conn) Login(qrChan chan<- string) (Session, error) {
 	}
 
 	session.ClientId = base64.StdEncoding.EncodeToString(clientId)
-	login := []interface{}{"admin", "init", waVersion, []string{wac.longClientName, wac.shortClientName}, session.ClientId, true}
+	login := []interface{}{"admin", "init", waVersion, []string{wac.longClientName, wac.shortClientName, wac.clientVersion}, session.ClientId, true}
 	loginChan, err := wac.writeJson(login)
 	if err != nil {
 		return session, fmt.Errorf("error writing login: %v\n", err)
@@ -231,7 +231,12 @@ func (wac *Conn) Login(qrChan chan<- string) (Session, error) {
 		return session, fmt.Errorf("error decoding login resp: %v\n", err)
 	}
 
-	ref := resp["ref"].(string)
+	var ref string
+	if rref, ok := resp["ref"].(string); ok {
+		ref = rref
+	} else {
+		return session, fmt.Errorf("error decoding login resp: invalid resp['ref']\n")
+	}
 
 	priv, pub, err := curve25519.GenerateKey()
 	if err != nil {
@@ -369,7 +374,7 @@ func (wac *Conn) Restore() error {
 	wac.listener.Unlock()
 
 	//admin init
-	init := []interface{}{"admin", "init", waVersion, []string{wac.longClientName, wac.shortClientName}, wac.session.ClientId, true}
+	init := []interface{}{"admin", "init", waVersion, []string{wac.longClientName, wac.shortClientName, wac.clientVersion}, wac.session.ClientId, true}
 	initChan, err := wac.writeJson(init)
 	if err != nil {
 		return fmt.Errorf("error writing admin init: %v\n", err)
@@ -390,9 +395,11 @@ func (wac *Conn) Restore() error {
 		}
 
 		if int(resp["status"].(float64)) != 200 {
+			wac.timeTag = ""
 			return fmt.Errorf("init responded with %d", resp["status"])
 		}
 	case <-time.After(wac.msgTimeout):
+		wac.timeTag = ""
 		return fmt.Errorf("restore session init timed out")
 	}
 
@@ -401,10 +408,11 @@ func (wac *Conn) Restore() error {
 	select {
 	case r1 := <-s1:
 		if err := json.Unmarshal([]byte(r1), &connResp); err != nil {
+			wac.timeTag = ""
 			return fmt.Errorf("error decoding s1 message: %v\n", err)
 		}
 	case <-time.After(wac.msgTimeout):
-
+		wac.timeTag = ""
 		//check for an error message
 		select {
 		case r := <-loginChan:
@@ -429,15 +437,18 @@ func (wac *Conn) Restore() error {
 		wac.listener.Unlock()
 
 		if err := wac.resolveChallenge(connResp[1].(map[string]interface{})["challenge"].(string)); err != nil {
+			wac.timeTag = ""
 			return fmt.Errorf("error resolving challenge: %v\n", err)
 		}
 
 		select {
 		case r := <-s2:
 			if err := json.Unmarshal([]byte(r), &connResp); err != nil {
+				wac.timeTag = ""
 				return fmt.Errorf("error decoding s2 message: %v\n", err)
 			}
 		case <-time.After(wac.msgTimeout):
+			wac.timeTag = ""
 			return fmt.Errorf("restore session challenge timed out")
 		}
 	}
@@ -447,13 +458,16 @@ func (wac *Conn) Restore() error {
 	case r := <-loginChan:
 		var resp map[string]interface{}
 		if err = json.Unmarshal([]byte(r), &resp); err != nil {
+			wac.timeTag = ""
 			return fmt.Errorf("error decoding login connResp: %v\n", err)
 		}
 
 		if int(resp["status"].(float64)) != 200 {
+			wac.timeTag = ""
 			return fmt.Errorf("admin login responded with %d", resp["status"])
 		}
 	case <-time.After(wac.msgTimeout):
+		wac.timeTag = ""
 		return fmt.Errorf("restore session login timed out")
 	}
 

vendor/github.com/Rhymen/go-whatsapp/write.go

@@ -30,6 +30,11 @@ func (wac *Conn) writeJson(data []interface{}) (<-chan string, error) {
 	messageTag := fmt.Sprintf("%d.--%d", ts, wac.msgCount)
 	bytes := []byte(fmt.Sprintf("%s,%s", messageTag, d))
 
+	if wac.timeTag == "" {
+		tss := fmt.Sprintf("%d", ts)
+		wac.timeTag = tss[len(tss)-3:]
+	}
+
 	ch, err := wac.write(websocket.TextMessage, messageTag, bytes)
 	if err != nil {
 		return nil, err
@@ -127,6 +132,9 @@ func (wac *Conn) write(messageType int, answerMessageTag string, data []byte) (<
 		wac.listener.Unlock()
 	}
 
+	if wac == nil || wac.ws == nil {
+		return nil, ErrInvalidWebsocket
+	}
 	wac.ws.Lock()
 	err := wac.ws.conn.WriteMessage(messageType, data)
 	wac.ws.Unlock()

vendor/github.com/blang/semver/.travis.yml

@@ -0,0 +1,21 @@
+language: go
+matrix:
+  include:
+  - go: 1.4.3
+  - go: 1.5.4
+  - go: 1.6.3
+  - go: 1.7
+  - go: tip
+  allow_failures:
+  - go: tip
+install:
+- go get golang.org/x/tools/cmd/cover
+- go get github.com/mattn/goveralls
+script:
+- echo "Test and track coverage" ; $HOME/gopath/bin/goveralls -package "." -service=travis-ci
+  -repotoken $COVERALLS_TOKEN
+- echo "Build examples" ; cd examples && go build
+- echo "Check if gofmt'd" ; diff -u <(echo -n) <(gofmt -d -s .)
+env:
+  global:
+    secure: HroGEAUQpVq9zX1b1VIkraLiywhGbzvNnTZq2TMxgK7JHP8xqNplAeF1izrR2i4QLL9nsY+9WtYss4QuPvEtZcVHUobw6XnL6radF7jS1LgfYZ9Y7oF+zogZ2I5QUMRLGA7rcxQ05s7mKq3XZQfeqaNts4bms/eZRefWuaFZbkw=

vendor/github.com/blang/semver/LICENSE

@@ -0,0 +1,22 @@
+The MIT License
+
+Copyright (c) 2014 Benedikt Lang <github at benediktlang.de>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+

vendor/github.com/blang/semver/README.md

@@ -0,0 +1,194 @@
+semver for golang [![Build Status](https://travis-ci.org/blang/semver.svg?branch=master)](https://travis-ci.org/blang/semver) [![GoDoc](https://godoc.org/github.com/blang/semver?status.png)](https://godoc.org/github.com/blang/semver) [![Coverage Status](https://img.shields.io/coveralls/blang/semver.svg)](https://coveralls.io/r/blang/semver?branch=master)
+======
+
+semver is a [Semantic Versioning](http://semver.org/) library written in golang. It fully covers spec version `2.0.0`.
+
+Usage
+-----
+```bash
+$ go get github.com/blang/semver
+```
+Note: Always vendor your dependencies or fix on a specific version tag.
+
+```go
+import github.com/blang/semver
+v1, err := semver.Make("1.0.0-beta")
+v2, err := semver.Make("2.0.0-beta")
+v1.Compare(v2)
+```
+
+Also check the [GoDocs](http://godoc.org/github.com/blang/semver).
+
+Why should I use this lib?
+-----
+
+- Fully spec compatible
+- No reflection
+- No regex
+- Fully tested (Coverage >99%)
+- Readable parsing/validation errors
+- Fast (See [Benchmarks](#benchmarks))
+- Only Stdlib
+- Uses values instead of pointers
+- Many features, see below
+
+
+Features
+-----
+
+- Parsing and validation at all levels
+- Comparator-like comparisons
+- Compare Helper Methods
+- InPlace manipulation
+- Ranges `>=1.0.0 <2.0.0 || >=3.0.0 !3.0.1-beta.1`
+- Wildcards `>=1.x`, `<=2.5.x`
+- Sortable (implements sort.Interface)
+- database/sql compatible (sql.Scanner/Valuer)
+- encoding/json compatible (json.Marshaler/Unmarshaler)
+
+Ranges
+------
+
+A `Range` is a set of conditions which specify which versions satisfy the range.
+
+A condition is composed of an operator and a version. The supported operators are:
+
+- `<1.0.0` Less than `1.0.0`
+- `<=1.0.0` Less than or equal to `1.0.0`
+- `>1.0.0` Greater than `1.0.0`
+- `>=1.0.0` Greater than or equal to `1.0.0`
+- `1.0.0`, `=1.0.0`, `==1.0.0` Equal to `1.0.0`
+- `!1.0.0`, `!=1.0.0` Not equal to `1.0.0`. Excludes version `1.0.0`.
+
+Note that spaces between the operator and the version will be gracefully tolerated.
+
+A `Range` can link multiple `Ranges` separated by space:
+
+Ranges can be linked by logical AND:
+
+  - `>1.0.0 <2.0.0` would match between both ranges, so `1.1.1` and `1.8.7` but not `1.0.0` or `2.0.0`
+  - `>1.0.0 <3.0.0 !2.0.3-beta.2` would match every version between `1.0.0` and `3.0.0` except `2.0.3-beta.2`
+
+Ranges can also be linked by logical OR:
+
+  - `<2.0.0 || >=3.0.0` would match `1.x.x` and `3.x.x` but not `2.x.x`
+
+AND has a higher precedence than OR. It's not possible to use brackets.
+
+Ranges can be combined by both AND and OR
+
+  - `>1.0.0 <2.0.0 || >3.0.0 !4.2.1` would match `1.2.3`, `1.9.9`, `3.1.1`, but not `4.2.1`, `2.1.1`
+
+Range usage:
+
+```
+v, err := semver.Parse("1.2.3")
+range, err := semver.ParseRange(">1.0.0 <2.0.0 || >=3.0.0")
+if range(v) {
+    //valid
+}
+
+```
+
+Example
+-----
+
+Have a look at full examples in [examples/main.go](examples/main.go)
+
+```go
+import github.com/blang/semver
+
+v, err := semver.Make("0.0.1-alpha.preview+123.github")
+fmt.Printf("Major: %d\n", v.Major)
+fmt.Printf("Minor: %d\n", v.Minor)
+fmt.Printf("Patch: %d\n", v.Patch)
+fmt.Printf("Pre: %s\n", v.Pre)
+fmt.Printf("Build: %s\n", v.Build)
+
+// Prerelease versions array
+if len(v.Pre) > 0 {
+    fmt.Println("Prerelease versions:")
+    for i, pre := range v.Pre {
+        fmt.Printf("%d: %q\n", i, pre)
+    }
+}
+
+// Build meta data array
+if len(v.Build) > 0 {
+    fmt.Println("Build meta data:")
+    for i, build := range v.Build {
+        fmt.Printf("%d: %q\n", i, build)
+    }
+}
+
+v001, err := semver.Make("0.0.1")
+// Compare using helpers: v.GT(v2), v.LT, v.GTE, v.LTE
+v001.GT(v) == true
+v.LT(v001) == true
+v.GTE(v) == true
+v.LTE(v) == true
+
+// Or use v.Compare(v2) for comparisons (-1, 0, 1):
+v001.Compare(v) == 1
+v.Compare(v001) == -1
+v.Compare(v) == 0
+
+// Manipulate Version in place:
+v.Pre[0], err = semver.NewPRVersion("beta")
+if err != nil {
+    fmt.Printf("Error parsing pre release version: %q", err)
+}
+
+fmt.Println("\nValidate versions:")
+v.Build[0] = "?"
+
+err = v.Validate()
+if err != nil {
+    fmt.Printf("Validation failed: %s\n", err)
+}
+```
+
+
+Benchmarks
+-----
+
+    BenchmarkParseSimple-4           5000000    390    ns/op    48 B/op   1 allocs/op
+    BenchmarkParseComplex-4          1000000   1813    ns/op   256 B/op   7 allocs/op
+    BenchmarkParseAverage-4          1000000   1171    ns/op   163 B/op   4 allocs/op
+    BenchmarkStringSimple-4         20000000    119    ns/op    16 B/op   1 allocs/op
+    BenchmarkStringLarger-4         10000000    206    ns/op    32 B/op   2 allocs/op
+    BenchmarkStringComplex-4         5000000    324    ns/op    80 B/op   3 allocs/op
+    BenchmarkStringAverage-4         5000000    273    ns/op    53 B/op   2 allocs/op
+    BenchmarkValidateSimple-4      200000000      9.33 ns/op     0 B/op   0 allocs/op
+    BenchmarkValidateComplex-4       3000000    469    ns/op     0 B/op   0 allocs/op
+    BenchmarkValidateAverage-4       5000000    256    ns/op     0 B/op   0 allocs/op
+    BenchmarkCompareSimple-4       100000000     11.8  ns/op     0 B/op   0 allocs/op
+    BenchmarkCompareComplex-4       50000000     30.8  ns/op     0 B/op   0 allocs/op
+    BenchmarkCompareAverage-4       30000000     41.5  ns/op     0 B/op   0 allocs/op
+    BenchmarkSort-4                  3000000    419    ns/op   256 B/op   2 allocs/op
+    BenchmarkRangeParseSimple-4      2000000    850    ns/op   192 B/op   5 allocs/op
+    BenchmarkRangeParseAverage-4     1000000   1677    ns/op   400 B/op  10 allocs/op
+    BenchmarkRangeParseComplex-4      300000   5214    ns/op  1440 B/op  30 allocs/op
+    BenchmarkRangeMatchSimple-4     50000000     25.6  ns/op     0 B/op   0 allocs/op
+    BenchmarkRangeMatchAverage-4    30000000     56.4  ns/op     0 B/op   0 allocs/op
+    BenchmarkRangeMatchComplex-4    10000000    153    ns/op     0 B/op   0 allocs/op
+
+See benchmark cases at [semver_test.go](semver_test.go)
+
+
+Motivation
+-----
+
+I simply couldn't find any lib supporting the full spec. Others were just wrong or used reflection and regex which i don't like.
+
+
+Contribution
+-----
+
+Feel free to make a pull request. For bigger changes create a issue first to discuss about it.
+
+
+License
+-----
+
+See [LICENSE](LICENSE) file.

vendor/github.com/blang/semver/json.go

@@ -0,0 +1,23 @@
+package semver
+
+import (
+	"encoding/json"
+)
+
+// MarshalJSON implements the encoding/json.Marshaler interface.
+func (v Version) MarshalJSON() ([]byte, error) {
+	return json.Marshal(v.String())
+}
+
+// UnmarshalJSON implements the encoding/json.Unmarshaler interface.
+func (v *Version) UnmarshalJSON(data []byte) (err error) {
+	var versionString string
+
+	if err = json.Unmarshal(data, &versionString); err != nil {
+		return
+	}
+
+	*v, err = Parse(versionString)
+
+	return
+}

vendor/github.com/blang/semver/package.json

@@ -0,0 +1,17 @@
+{
+  "author": "blang",
+  "bugs": {
+    "URL": "https://github.com/blang/semver/issues",
+    "url": "https://github.com/blang/semver/issues"
+  },
+  "gx": {
+    "dvcsimport": "github.com/blang/semver"
+  },
+  "gxVersion": "0.10.0",
+  "language": "go",
+  "license": "MIT",
+  "name": "semver",
+  "releaseCmd": "git commit -a -m \"gx publish $VERSION\"",
+  "version": "3.5.1"
+}
+

vendor/github.com/blang/semver/range.go

@@ -0,0 +1,416 @@
+package semver
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+	"unicode"
+)
+
+type wildcardType int
+
+const (
+	noneWildcard  wildcardType = iota
+	majorWildcard wildcardType = 1
+	minorWildcard wildcardType = 2
+	patchWildcard wildcardType = 3
+)
+
+func wildcardTypefromInt(i int) wildcardType {
+	switch i {
+	case 1:
+		return majorWildcard
+	case 2:
+		return minorWildcard
+	case 3:
+		return patchWildcard
+	default:
+		return noneWildcard
+	}
+}
+
+type comparator func(Version, Version) bool
+
+var (
+	compEQ comparator = func(v1 Version, v2 Version) bool {
+		return v1.Compare(v2) == 0
+	}
+	compNE = func(v1 Version, v2 Version) bool {
+		return v1.Compare(v2) != 0
+	}
+	compGT = func(v1 Version, v2 Version) bool {
+		return v1.Compare(v2) == 1
+	}
+	compGE = func(v1 Version, v2 Version) bool {
+		return v1.Compare(v2) >= 0
+	}
+	compLT = func(v1 Version, v2 Version) bool {
+		return v1.Compare(v2) == -1
+	}
+	compLE = func(v1 Version, v2 Version) bool {
+		return v1.Compare(v2) <= 0
+	}
+)
+
+type versionRange struct {
+	v Version
+	c comparator
+}
+
+// rangeFunc creates a Range from the given versionRange.
+func (vr *versionRange) rangeFunc() Range {
+	return Range(func(v Version) bool {
+		return vr.c(v, vr.v)
+	})
+}
+
+// Range represents a range of versions.
+// A Range can be used to check if a Version satisfies it:
+//
+//     range, err := semver.ParseRange(">1.0.0 <2.0.0")
+//     range(semver.MustParse("1.1.1") // returns true
+type Range func(Version) bool
+
+// OR combines the existing Range with another Range using logical OR.
+func (rf Range) OR(f Range) Range {
+	return Range(func(v Version) bool {
+		return rf(v) || f(v)
+	})
+}
+
+// AND combines the existing Range with another Range using logical AND.
+func (rf Range) AND(f Range) Range {
+	return Range(func(v Version) bool {
+		return rf(v) && f(v)
+	})
+}
+
+// ParseRange parses a range and returns a Range.
+// If the range could not be parsed an error is returned.
+//
+// Valid ranges are:
+//   - "<1.0.0"
+//   - "<=1.0.0"
+//   - ">1.0.0"
+//   - ">=1.0.0"
+//   - "1.0.0", "=1.0.0", "==1.0.0"
+//   - "!1.0.0", "!=1.0.0"
+//
+// A Range can consist of multiple ranges separated by space:
+// Ranges can be linked by logical AND:
+//   - ">1.0.0 <2.0.0" would match between both ranges, so "1.1.1" and "1.8.7" but not "1.0.0" or "2.0.0"
+//   - ">1.0.0 <3.0.0 !2.0.3-beta.2" would match every version between 1.0.0 and 3.0.0 except 2.0.3-beta.2
+//
+// Ranges can also be linked by logical OR:
+//   - "<2.0.0 || >=3.0.0" would match "1.x.x" and "3.x.x" but not "2.x.x"
+//
+// AND has a higher precedence than OR. It's not possible to use brackets.
+//
+// Ranges can be combined by both AND and OR
+//
+//  - `>1.0.0 <2.0.0 || >3.0.0 !4.2.1` would match `1.2.3`, `1.9.9`, `3.1.1`, but not `4.2.1`, `2.1.1`
+func ParseRange(s string) (Range, error) {
+	parts := splitAndTrim(s)
+	orParts, err := splitORParts(parts)
+	if err != nil {
+		return nil, err
+	}
+	expandedParts, err := expandWildcardVersion(orParts)
+	if err != nil {
+		return nil, err
+	}
+	var orFn Range
+	for _, p := range expandedParts {
+		var andFn Range
+		for _, ap := range p {
+			opStr, vStr, err := splitComparatorVersion(ap)
+			if err != nil {
+				return nil, err
+			}
+			vr, err := buildVersionRange(opStr, vStr)
+			if err != nil {
+				return nil, fmt.Errorf("Could not parse Range %q: %s", ap, err)
+			}
+			rf := vr.rangeFunc()
+
+			// Set function
+			if andFn == nil {
+				andFn = rf
+			} else { // Combine with existing function
+				andFn = andFn.AND(rf)
+			}
+		}
+		if orFn == nil {
+			orFn = andFn
+		} else {
+			orFn = orFn.OR(andFn)
+		}
+
+	}
+	return orFn, nil
+}
+
+// splitORParts splits the already cleaned parts by '||'.
+// Checks for invalid positions of the operator and returns an
+// error if found.
+func splitORParts(parts []string) ([][]string, error) {
+	var ORparts [][]string
+	last := 0
+	for i, p := range parts {
+		if p == "||" {
+			if i == 0 {
+				return nil, fmt.Errorf("First element in range is '||'")
+			}
+			ORparts = append(ORparts, parts[last:i])
+			last = i + 1
+		}
+	}
+	if last == len(parts) {
+		return nil, fmt.Errorf("Last element in range is '||'")
+	}
+	ORparts = append(ORparts, parts[last:])
+	return ORparts, nil
+}
+
+// buildVersionRange takes a slice of 2: operator and version
+// and builds a versionRange, otherwise an error.
+func buildVersionRange(opStr, vStr string) (*versionRange, error) {
+	c := parseComparator(opStr)
+	if c == nil {
+		return nil, fmt.Errorf("Could not parse comparator %q in %q", opStr, strings.Join([]string{opStr, vStr}, ""))
+	}
+	v, err := Parse(vStr)
+	if err != nil {
+		return nil, fmt.Errorf("Could not parse version %q in %q: %s", vStr, strings.Join([]string{opStr, vStr}, ""), err)
+	}
+
+	return &versionRange{
+		v: v,
+		c: c,
+	}, nil
+
+}
+
+// inArray checks if a byte is contained in an array of bytes
+func inArray(s byte, list []byte) bool {
+	for _, el := range list {
+		if el == s {
+			return true
+		}
+	}
+	return false
+}
+
+// splitAndTrim splits a range string by spaces and cleans whitespaces
+func splitAndTrim(s string) (result []string) {
+	last := 0
+	var lastChar byte
+	excludeFromSplit := []byte{'>', '<', '='}
+	for i := 0; i < len(s); i++ {
+		if s[i] == ' ' && !inArray(lastChar, excludeFromSplit) {
+			if last < i-1 {
+				result = append(result, s[last:i])
+			}
+			last = i + 1
+		} else if s[i] != ' ' {
+			lastChar = s[i]
+		}
+	}
+	if last < len(s)-1 {
+		result = append(result, s[last:])
+	}
+
+	for i, v := range result {
+		result[i] = strings.Replace(v, " ", "", -1)
+	}
+
+	// parts := strings.Split(s, " ")
+	// for _, x := range parts {
+	// 	if s := strings.TrimSpace(x); len(s) != 0 {
+	// 		result = append(result, s)
+	// 	}
+	// }
+	return
+}
+
+// splitComparatorVersion splits the comparator from the version.
+// Input must be free of leading or trailing spaces.
+func splitComparatorVersion(s string) (string, string, error) {
+	i := strings.IndexFunc(s, unicode.IsDigit)
+	if i == -1 {
+		return "", "", fmt.Errorf("Could not get version from string: %q", s)
+	}
+	return strings.TrimSpace(s[0:i]), s[i:], nil
+}
+
+// getWildcardType will return the type of wildcard that the
+// passed version contains
+func getWildcardType(vStr string) wildcardType {
+	parts := strings.Split(vStr, ".")
+	nparts := len(parts)
+	wildcard := parts[nparts-1]
+
+	possibleWildcardType := wildcardTypefromInt(nparts)
+	if wildcard == "x" {
+		return possibleWildcardType
+	}
+
+	return noneWildcard
+}
+
+// createVersionFromWildcard will convert a wildcard version
+// into a regular version, replacing 'x's with '0's, handling
+// special cases like '1.x.x' and '1.x'
+func createVersionFromWildcard(vStr string) string {
+	// handle 1.x.x
+	vStr2 := strings.Replace(vStr, ".x.x", ".x", 1)
+	vStr2 = strings.Replace(vStr2, ".x", ".0", 1)
+	parts := strings.Split(vStr2, ".")
+
+	// handle 1.x
+	if len(parts) == 2 {
+		return vStr2 + ".0"
+	}
+
+	return vStr2
+}
+
+// incrementMajorVersion will increment the major version
+// of the passed version
+func incrementMajorVersion(vStr string) (string, error) {
+	parts := strings.Split(vStr, ".")
+	i, err := strconv.Atoi(parts[0])
+	if err != nil {
+		return "", err
+	}
+	parts[0] = strconv.Itoa(i + 1)
+
+	return strings.Join(parts, "."), nil
+}
+
+// incrementMajorVersion will increment the minor version
+// of the passed version
+func incrementMinorVersion(vStr string) (string, error) {
+	parts := strings.Split(vStr, ".")
+	i, err := strconv.Atoi(parts[1])
+	if err != nil {
+		return "", err
+	}
+	parts[1] = strconv.Itoa(i + 1)
+
+	return strings.Join(parts, "."), nil
+}
+
+// expandWildcardVersion will expand wildcards inside versions
+// following these rules:
+//
+// * when dealing with patch wildcards:
+// >= 1.2.x    will become    >= 1.2.0
+// <= 1.2.x    will become    <  1.3.0
+// >  1.2.x    will become    >= 1.3.0
+// <  1.2.x    will become    <  1.2.0
+// != 1.2.x    will become    <  1.2.0 >= 1.3.0
+//
+// * when dealing with minor wildcards:
+// >= 1.x      will become    >= 1.0.0
+// <= 1.x      will become    <  2.0.0
+// >  1.x      will become    >= 2.0.0
+// <  1.0      will become    <  1.0.0
+// != 1.x      will become    <  1.0.0 >= 2.0.0
+//
+// * when dealing with wildcards without
+// version operator:
+// 1.2.x       will become    >= 1.2.0 < 1.3.0
+// 1.x         will become    >= 1.0.0 < 2.0.0
+func expandWildcardVersion(parts [][]string) ([][]string, error) {
+	var expandedParts [][]string
+	for _, p := range parts {
+		var newParts []string
+		for _, ap := range p {
+			if strings.Index(ap, "x") != -1 {
+				opStr, vStr, err := splitComparatorVersion(ap)
+				if err != nil {
+					return nil, err
+				}
+
+				versionWildcardType := getWildcardType(vStr)
+				flatVersion := createVersionFromWildcard(vStr)
+
+				var resultOperator string
+				var shouldIncrementVersion bool
+				switch opStr {
+				case ">":
+					resultOperator = ">="
+					shouldIncrementVersion = true
+				case ">=":
+					resultOperator = ">="
+				case "<":
+					resultOperator = "<"
+				case "<=":
+					resultOperator = "<"
+					shouldIncrementVersion = true
+				case "", "=", "==":
+					newParts = append(newParts, ">="+flatVersion)
+					resultOperator = "<"
+					shouldIncrementVersion = true
+				case "!=", "!":
+					newParts = append(newParts, "<"+flatVersion)
+					resultOperator = ">="
+					shouldIncrementVersion = true
+				}
+
+				var resultVersion string
+				if shouldIncrementVersion {
+					switch versionWildcardType {
+					case patchWildcard:
+						resultVersion, _ = incrementMinorVersion(flatVersion)
+					case minorWildcard:
+						resultVersion, _ = incrementMajorVersion(flatVersion)
+					}
+				} else {
+					resultVersion = flatVersion
+				}
+
+				ap = resultOperator + resultVersion
+			}
+			newParts = append(newParts, ap)
+		}
+		expandedParts = append(expandedParts, newParts)
+	}
+
+	return expandedParts, nil
+}
+
+func parseComparator(s string) comparator {
+	switch s {
+	case "==":
+		fallthrough
+	case "":
+		fallthrough
+	case "=":
+		return compEQ
+	case ">":
+		return compGT
+	case ">=":
+		return compGE
+	case "<":
+		return compLT
+	case "<=":
+		return compLE
+	case "!":
+		fallthrough
+	case "!=":
+		return compNE
+	}
+
+	return nil
+}
+
+// MustParseRange is like ParseRange but panics if the range cannot be parsed.
+func MustParseRange(s string) Range {
+	r, err := ParseRange(s)
+	if err != nil {
+		panic(`semver: ParseRange(` + s + `): ` + err.Error())
+	}
+	return r
+}

vendor/github.com/blang/semver/semver.go

@@ -0,0 +1,418 @@
+package semver
+
+import (
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+)
+
+const (
+	numbers  string = "0123456789"
+	alphas          = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-"
+	alphanum        = alphas + numbers
+)
+
+// SpecVersion is the latest fully supported spec version of semver
+var SpecVersion = Version{
+	Major: 2,
+	Minor: 0,
+	Patch: 0,
+}
+
+// Version represents a semver compatible version
+type Version struct {
+	Major uint64
+	Minor uint64
+	Patch uint64
+	Pre   []PRVersion
+	Build []string //No Precendence
+}
+
+// Version to string
+func (v Version) String() string {
+	b := make([]byte, 0, 5)
+	b = strconv.AppendUint(b, v.Major, 10)
+	b = append(b, '.')
+	b = strconv.AppendUint(b, v.Minor, 10)
+	b = append(b, '.')
+	b = strconv.AppendUint(b, v.Patch, 10)
+
+	if len(v.Pre) > 0 {
+		b = append(b, '-')
+		b = append(b, v.Pre[0].String()...)
+
+		for _, pre := range v.Pre[1:] {
+			b = append(b, '.')
+			b = append(b, pre.String()...)
+		}
+	}
+
+	if len(v.Build) > 0 {
+		b = append(b, '+')
+		b = append(b, v.Build[0]...)
+
+		for _, build := range v.Build[1:] {
+			b = append(b, '.')
+			b = append(b, build...)
+		}
+	}
+
+	return string(b)
+}
+
+// Equals checks if v is equal to o.
+func (v Version) Equals(o Version) bool {
+	return (v.Compare(o) == 0)
+}
+
+// EQ checks if v is equal to o.
+func (v Version) EQ(o Version) bool {
+	return (v.Compare(o) == 0)
+}
+
+// NE checks if v is not equal to o.
+func (v Version) NE(o Version) bool {
+	return (v.Compare(o) != 0)
+}
+
+// GT checks if v is greater than o.
+func (v Version) GT(o Version) bool {
+	return (v.Compare(o) == 1)
+}
+
+// GTE checks if v is greater than or equal to o.
+func (v Version) GTE(o Version) bool {
+	return (v.Compare(o) >= 0)
+}
+
+// GE checks if v is greater than or equal to o.
+func (v Version) GE(o Version) bool {
+	return (v.Compare(o) >= 0)
+}
+
+// LT checks if v is less than o.
+func (v Version) LT(o Version) bool {
+	return (v.Compare(o) == -1)
+}
+
+// LTE checks if v is less than or equal to o.
+func (v Version) LTE(o Version) bool {
+	return (v.Compare(o) <= 0)
+}
+
+// LE checks if v is less than or equal to o.
+func (v Version) LE(o Version) bool {
+	return (v.Compare(o) <= 0)
+}
+
+// Compare compares Versions v to o:
+// -1 == v is less than o
+// 0 == v is equal to o
+// 1 == v is greater than o
+func (v Version) Compare(o Version) int {
+	if v.Major != o.Major {
+		if v.Major > o.Major {
+			return 1
+		}
+		return -1
+	}
+	if v.Minor != o.Minor {
+		if v.Minor > o.Minor {
+			return 1
+		}
+		return -1
+	}
+	if v.Patch != o.Patch {
+		if v.Patch > o.Patch {
+			return 1
+		}
+		return -1
+	}
+
+	// Quick comparison if a version has no prerelease versions
+	if len(v.Pre) == 0 && len(o.Pre) == 0 {
+		return 0
+	} else if len(v.Pre) == 0 && len(o.Pre) > 0 {
+		return 1
+	} else if len(v.Pre) > 0 && len(o.Pre) == 0 {
+		return -1
+	}
+
+	i := 0
+	for ; i < len(v.Pre) && i < len(o.Pre); i++ {
+		if comp := v.Pre[i].Compare(o.Pre[i]); comp == 0 {
+			continue
+		} else if comp == 1 {
+			return 1
+		} else {
+			return -1
+		}
+	}
+
+	// If all pr versions are the equal but one has further prversion, this one greater
+	if i == len(v.Pre) && i == len(o.Pre) {
+		return 0
+	} else if i == len(v.Pre) && i < len(o.Pre) {
+		return -1
+	} else {
+		return 1
+	}
+
+}
+
+// Validate validates v and returns error in case
+func (v Version) Validate() error {
+	// Major, Minor, Patch already validated using uint64
+
+	for _, pre := range v.Pre {
+		if !pre.IsNum { //Numeric prerelease versions already uint64
+			if len(pre.VersionStr) == 0 {
+				return fmt.Errorf("Prerelease can not be empty %q", pre.VersionStr)
+			}
+			if !containsOnly(pre.VersionStr, alphanum) {
+				return fmt.Errorf("Invalid character(s) found in prerelease %q", pre.VersionStr)
+			}
+		}
+	}
+
+	for _, build := range v.Build {
+		if len(build) == 0 {
+			return fmt.Errorf("Build meta data can not be empty %q", build)
+		}
+		if !containsOnly(build, alphanum) {
+			return fmt.Errorf("Invalid character(s) found in build meta data %q", build)
+		}
+	}
+
+	return nil
+}
+
+// New is an alias for Parse and returns a pointer, parses version string and returns a validated Version or error
+func New(s string) (vp *Version, err error) {
+	v, err := Parse(s)
+	vp = &v
+	return
+}
+
+// Make is an alias for Parse, parses version string and returns a validated Version or error
+func Make(s string) (Version, error) {
+	return Parse(s)
+}
+
+// ParseTolerant allows for certain version specifications that do not strictly adhere to semver
+// specs to be parsed by this library. It does so by normalizing versions before passing them to
+// Parse(). It currently trims spaces, removes a "v" prefix, and adds a 0 patch number to versions
+// with only major and minor components specified
+func ParseTolerant(s string) (Version, error) {
+	s = strings.TrimSpace(s)
+	s = strings.TrimPrefix(s, "v")
+
+	// Split into major.minor.(patch+pr+meta)
+	parts := strings.SplitN(s, ".", 3)
+	if len(parts) < 3 {
+		if strings.ContainsAny(parts[len(parts)-1], "+-") {
+			return Version{}, errors.New("Short version cannot contain PreRelease/Build meta data")
+		}
+		for len(parts) < 3 {
+			parts = append(parts, "0")
+		}
+		s = strings.Join(parts, ".")
+	}
+
+	return Parse(s)
+}
+
+// Parse parses version string and returns a validated Version or error
+func Parse(s string) (Version, error) {
+	if len(s) == 0 {
+		return Version{}, errors.New("Version string empty")
+	}
+
+	// Split into major.minor.(patch+pr+meta)
+	parts := strings.SplitN(s, ".", 3)
+	if len(parts) != 3 {
+		return Version{}, errors.New("No Major.Minor.Patch elements found")
+	}
+
+	// Major
+	if !containsOnly(parts[0], numbers) {
+		return Version{}, fmt.Errorf("Invalid character(s) found in major number %q", parts[0])
+	}
+	if hasLeadingZeroes(parts[0]) {
+		return Version{}, fmt.Errorf("Major number must not contain leading zeroes %q", parts[0])
+	}
+	major, err := strconv.ParseUint(parts[0], 10, 64)
+	if err != nil {
+		return Version{}, err
+	}
+
+	// Minor
+	if !containsOnly(parts[1], numbers) {
+		return Version{}, fmt.Errorf("Invalid character(s) found in minor number %q", parts[1])
+	}
+	if hasLeadingZeroes(parts[1]) {
+		return Version{}, fmt.Errorf("Minor number must not contain leading zeroes %q", parts[1])
+	}
+	minor, err := strconv.ParseUint(parts[1], 10, 64)
+	if err != nil {
+		return Version{}, err
+	}
+
+	v := Version{}
+	v.Major = major
+	v.Minor = minor
+
+	var build, prerelease []string
+	patchStr := parts[2]
+
+	if buildIndex := strings.IndexRune(patchStr, '+'); buildIndex != -1 {
+		build = strings.Split(patchStr[buildIndex+1:], ".")
+		patchStr = patchStr[:buildIndex]
+	}
+
+	if preIndex := strings.IndexRune(patchStr, '-'); preIndex != -1 {
+		prerelease = strings.Split(patchStr[preIndex+1:], ".")
+		patchStr = patchStr[:preIndex]
+	}
+
+	if !containsOnly(patchStr, numbers) {
+		return Version{}, fmt.Errorf("Invalid character(s) found in patch number %q", patchStr)
+	}
+	if hasLeadingZeroes(patchStr) {
+		return Version{}, fmt.Errorf("Patch number must not contain leading zeroes %q", patchStr)
+	}
+	patch, err := strconv.ParseUint(patchStr, 10, 64)
+	if err != nil {
+		return Version{}, err
+	}
+
+	v.Patch = patch
+
+	// Prerelease
+	for _, prstr := range prerelease {
+		parsedPR, err := NewPRVersion(prstr)
+		if err != nil {
+			return Version{}, err
+		}
+		v.Pre = append(v.Pre, parsedPR)
+	}
+
+	// Build meta data
+	for _, str := range build {
+		if len(str) == 0 {
+			return Version{}, errors.New("Build meta data is empty")
+		}
+		if !containsOnly(str, alphanum) {
+			return Version{}, fmt.Errorf("Invalid character(s) found in build meta data %q", str)
+		}
+		v.Build = append(v.Build, str)
+	}
+
+	return v, nil
+}
+
+// MustParse is like Parse but panics if the version cannot be parsed.
+func MustParse(s string) Version {
+	v, err := Parse(s)
+	if err != nil {
+		panic(`semver: Parse(` + s + `): ` + err.Error())
+	}
+	return v
+}
+
+// PRVersion represents a PreRelease Version
+type PRVersion struct {
+	VersionStr string
+	VersionNum uint64
+	IsNum      bool
+}
+
+// NewPRVersion creates a new valid prerelease version
+func NewPRVersion(s string) (PRVersion, error) {
+	if len(s) == 0 {
+		return PRVersion{}, errors.New("Prerelease is empty")
+	}
+	v := PRVersion{}
+	if containsOnly(s, numbers) {
+		if hasLeadingZeroes(s) {
+			return PRVersion{}, fmt.Errorf("Numeric PreRelease version must not contain leading zeroes %q", s)
+		}
+		num, err := strconv.ParseUint(s, 10, 64)
+
+		// Might never be hit, but just in case
+		if err != nil {
+			return PRVersion{}, err
+		}
+		v.VersionNum = num
+		v.IsNum = true
+	} else if containsOnly(s, alphanum) {
+		v.VersionStr = s
+		v.IsNum = false
+	} else {
+		return PRVersion{}, fmt.Errorf("Invalid character(s) found in prerelease %q", s)
+	}
+	return v, nil
+}
+
+// IsNumeric checks if prerelease-version is numeric
+func (v PRVersion) IsNumeric() bool {
+	return v.IsNum
+}
+
+// Compare compares two PreRelease Versions v and o:
+// -1 == v is less than o
+// 0 == v is equal to o
+// 1 == v is greater than o
+func (v PRVersion) Compare(o PRVersion) int {
+	if v.IsNum && !o.IsNum {
+		return -1
+	} else if !v.IsNum && o.IsNum {
+		return 1
+	} else if v.IsNum && o.IsNum {
+		if v.VersionNum == o.VersionNum {
+			return 0
+		} else if v.VersionNum > o.VersionNum {
+			return 1
+		} else {
+			return -1
+		}
+	} else { // both are Alphas
+		if v.VersionStr == o.VersionStr {
+			return 0
+		} else if v.VersionStr > o.VersionStr {
+			return 1
+		} else {
+			return -1
+		}
+	}
+}
+
+// PreRelease version to string
+func (v PRVersion) String() string {
+	if v.IsNum {
+		return strconv.FormatUint(v.VersionNum, 10)
+	}
+	return v.VersionStr
+}
+
+func containsOnly(s string, set string) bool {
+	return strings.IndexFunc(s, func(r rune) bool {
+		return !strings.ContainsRune(set, r)
+	}) == -1
+}
+
+func hasLeadingZeroes(s string) bool {
+	return len(s) > 1 && s[0] == '0'
+}
+
+// NewBuildVersion creates a new valid build version
+func NewBuildVersion(s string) (string, error) {
+	if len(s) == 0 {
+		return "", errors.New("Buildversion is empty")
+	}
+	if !containsOnly(s, alphanum) {
+		return "", fmt.Errorf("Invalid character(s) found in build meta data %q", s)
+	}
+	return s, nil
+}

vendor/github.com/blang/semver/sort.go

@@ -0,0 +1,28 @@
+package semver
+
+import (
+	"sort"
+)
+
+// Versions represents multiple versions.
+type Versions []Version
+
+// Len returns length of version collection
+func (s Versions) Len() int {
+	return len(s)
+}
+
+// Swap swaps two versions inside the collection by its indices
+func (s Versions) Swap(i, j int) {
+	s[i], s[j] = s[j], s[i]
+}
+
+// Less checks if version at index i is less than version at index j
+func (s Versions) Less(i, j int) bool {
+	return s[i].LT(s[j])
+}
+
+// Sort sorts a slice of versions
+func Sort(versions []Version) {
+	sort.Sort(Versions(versions))
+}

vendor/github.com/blang/semver/sql.go

@@ -0,0 +1,30 @@
+package semver
+
+import (
+	"database/sql/driver"
+	"fmt"
+)
+
+// Scan implements the database/sql.Scanner interface.
+func (v *Version) Scan(src interface{}) (err error) {
+	var str string
+	switch src := src.(type) {
+	case string:
+		str = src
+	case []byte:
+		str = string(src)
+	default:
+		return fmt.Errorf("Version.Scan: cannot convert %T to string.", src)
+	}
+
+	if t, err := Parse(str); err == nil {
+		*v = t
+	}
+
+	return
+}
+
+// Value implements the database/sql/driver.Valuer interface.
+func (v Version) Value() (driver.Value, error) {
+	return v.String(), nil
+}

vendor/github.com/d5/tengo/v2/.goreleaser.yml

@@ -11,9 +11,10 @@ builds:
       - darwin
       - linux
       - windows
-archive:
-  files:
-    - none*
+archives:
+  -
+    files:
+      - none*
 checksum:
   name_template: 'checksums.txt'
 changelog:

vendor/github.com/d5/tengo/v2/Makefile

@@ -6,6 +6,7 @@ lint:
 
 test: generate lint
 	go test -race -cover ./...
+	go run ./cmd/tengo -resolve ./testdata/cli/test.tengo
 
 fmt:
 	go fmt ./...

vendor/github.com/d5/tengo/v2/README.md

@@ -5,9 +5,8 @@
 # The Tengo Language
 
 [![GoDoc](https://godoc.org/github.com/d5/tengo?status.svg)](https://godoc.org/github.com/d5/tengo)
+![test](https://github.com/d5/tengo/workflows/test/badge.svg)
 [![Go Report Card](https://goreportcard.com/badge/github.com/d5/tengo)](https://goreportcard.com/report/github.com/d5/tengo)
-[![CircleCI](https://circleci.com/gh/d5/tengo.svg?style=svg)](https://circleci.com/gh/d5/tengo)
-[![Sourcegraph](https://sourcegraph.com/github.com/d5/tengo/-/badge.svg)](https://sourcegraph.com/github.com/d5/tengo?badge)
 
 **Tengo is a small, dynamic, fast, secure script language for Go.** 
 
@@ -52,19 +51,21 @@ fmt.println(sum("", [1, 2, 3]))  // "123"
 
 ## Benchmark
 
-| | fib(35) | fibt(35) |  Type  |
+| | fib(35) | fibt(35) |  Language (Type)  |
 | :--- |    ---: |     ---: |  :---: |
-| Go | `48ms` | `3ms` | Go (native) |
-| [**Tengo**](https://github.com/d5/tengo) | `2,349ms` | `5ms` | VM on Go |
-| Lua | `1,416ms` | `3ms` | Lua (native) |
-| [go-lua](https://github.com/Shopify/go-lua) | `4,402ms` | `5ms` | Lua VM on Go |
-| [GopherLua](https://github.com/yuin/gopher-lua) | `4,023ms` | `5ms` | Lua VM on Go |
-| Python | `2,588ms` | `26ms` | Python (native) |
-| [starlark-go](https://github.com/google/starlark-go) | `11,126ms` | `6ms` | Python-like Interpreter on Go |
-| [gpython](https://github.com/go-python/gpython) | `15,035ms` | `4ms` | Python Interpreter on Go |
-| [goja](https://github.com/dop251/goja) | `5,089ms` | `5ms` | JS VM on Go |
-| [otto](https://github.com/robertkrimen/otto) | `68,377ms` | `11ms` | JS Interpreter on Go |
-| [Anko](https://github.com/mattn/anko) | `92,579ms` | `18ms` | Interpreter on Go |
+| [**Tengo**](https://github.com/d5/tengo) | `2,931ms` | `4ms` | Tengo (VM) |
+| [go-lua](https://github.com/Shopify/go-lua) | `4,824ms` | `4ms` | Lua (VM) |
+| [GopherLua](https://github.com/yuin/gopher-lua) | `5,365ms` | `4ms` | Lua (VM) |
+| [goja](https://github.com/dop251/goja) | `5,533ms` | `5ms` | JavaScript (VM) |
+| [starlark-go](https://github.com/google/starlark-go) | `11,495ms` | `5ms` | Starlark (Interpreter) |
+| [Yaegi](https://github.com/containous/yaegi) | `15,645ms` | `12ms` | Yaegi (Interpreter) |
+| [gpython](https://github.com/go-python/gpython) | `16,322ms` | `5ms` | Python (Interpreter) |
+| [otto](https://github.com/robertkrimen/otto) | `73,093ms` | `10ms` | JavaScript (Interpreter) |
+| [Anko](https://github.com/mattn/anko) | `79,809ms` | `8ms` | Anko (Interpreter) |
+| - | - | - | - |
+| Go | `53ms` | `3ms` | Go (Native) |
+| Lua | `1,612ms` | `3ms` | Lua (Native) |
+| Python | `2,632ms` | `23ms` | Python 2 (Native) |
 
 _* [fib(35)](https://github.com/d5/tengobench/blob/master/code/fib.tengo):
 Fibonacci(35)_  
@@ -75,6 +76,10 @@ _* See [here](https://github.com/d5/tengobench) for commands/codes used_
 
 ## Quick Start
 
+```
+go get github.com/d5/tengo/v2
+```
+
 A simple Go example code that compiles/runs Tengo script code with some input/output values:
 
 ```golang
@@ -133,3 +138,10 @@ each([a, b, c, d], func(x) {
 - [Interoperability](https://github.com/d5/tengo/blob/master/docs/interoperability.md)
 - [Tengo CLI](https://github.com/d5/tengo/blob/master/docs/tengo-cli.md)
 - [Standard Library](https://github.com/d5/tengo/blob/master/docs/stdlib.md)
+- Syntax Highlighters: [VSCode](https://github.com/lissein/vscode-tengo), [Atom](https://github.com/d5/tengo-atom)
+- **Why the name Tengo?** It's from [1Q84](https://en.wikipedia.org/wiki/1Q84).
+
+##
+
+:hearts: Like writing Go code? Come work at Skool. [We're hiring!](https://jobs.lever.co/skool)
+

vendor/github.com/d5/tengo/v2/builtins.go

@@ -13,6 +13,14 @@ var builtinFuncs = []*BuiltinFunction{
 		Name:  "append",
 		Value: builtinAppend,
 	},
+	{
+		Name:  "delete",
+		Value: builtinDelete,
+	},
+	{
+		Name:  "splice",
+		Value: builtinSplice,
+	},
 	{
 		Name:  "string",
 		Value: builtinString,
@@ -500,3 +508,104 @@ func builtinAppend(args ...Object) (Object, error) {
 		}
 	}
 }
+
+// builtinDelete deletes Map keys
+// usage: delete(map, "key")
+// key must be a string
+func builtinDelete(args ...Object) (Object, error) {
+	argsLen := len(args)
+	if argsLen != 2 {
+		return nil, ErrWrongNumArguments
+	}
+	switch arg := args[0].(type) {
+	case *Map:
+		if key, ok := args[1].(*String); ok {
+			delete(arg.Value, key.Value)
+			return UndefinedValue, nil
+		}
+		return nil, ErrInvalidArgumentType{
+			Name:     "second",
+			Expected: "string",
+			Found:    args[1].TypeName(),
+		}
+	default:
+		return nil, ErrInvalidArgumentType{
+			Name:     "first",
+			Expected: "map",
+			Found:    arg.TypeName(),
+		}
+	}
+}
+
+// builtinSplice deletes and changes given Array, returns deleted items.
+// usage:
+// deleted_items := splice(array[,start[,delete_count[,item1[,item2[,...]]]])
+func builtinSplice(args ...Object) (Object, error) {
+	argsLen := len(args)
+	if argsLen == 0 {
+		return nil, ErrWrongNumArguments
+	}
+
+	array, ok := args[0].(*Array)
+	if !ok {
+		return nil, ErrInvalidArgumentType{
+			Name:     "first",
+			Expected: "array",
+			Found:    args[0].TypeName(),
+		}
+	}
+	arrayLen := len(array.Value)
+
+	var startIdx int
+	if argsLen > 1 {
+		arg1, ok := args[1].(*Int)
+		if !ok {
+			return nil, ErrInvalidArgumentType{
+				Name:     "second",
+				Expected: "int",
+				Found:    args[1].TypeName(),
+			}
+		}
+		startIdx = int(arg1.Value)
+		if startIdx < 0 || startIdx > arrayLen {
+			return nil, ErrIndexOutOfBounds
+		}
+	}
+
+	delCount := len(array.Value)
+	if argsLen > 2 {
+		arg2, ok := args[2].(*Int)
+		if !ok {
+			return nil, ErrInvalidArgumentType{
+				Name:     "third",
+				Expected: "int",
+				Found:    args[2].TypeName(),
+			}
+		}
+		delCount = int(arg2.Value)
+		if delCount < 0 {
+			return nil, ErrIndexOutOfBounds
+		}
+	}
+	// if count of to be deleted items is bigger than expected, truncate it
+	if startIdx+delCount > arrayLen {
+		delCount = arrayLen - startIdx
+	}
+	// delete items
+	endIdx := startIdx + delCount
+	deleted := append([]Object{}, array.Value[startIdx:endIdx]...)
+
+	head := array.Value[:startIdx]
+	var items []Object
+	if argsLen > 3 {
+		items = make([]Object, 0, argsLen-3)
+		for i := 3; i < argsLen; i++ {
+			items = append(items, args[i])
+		}
+	}
+	items = append(items, array.Value[endIdx:]...)
+	array.Value = append(head, items...)
+
+	// return deleted items
+	return &Array{Value: deleted}, nil
+}

vendor/github.com/d5/tengo/v2/bytecode.go

@@ -97,6 +97,7 @@ func (b *Bytecode) RemoveDuplicates() {
 	var deduped []Object
 
 	indexMap := make(map[int]int) // mapping from old constant index to new index
+	fns := make(map[*CompiledFunction]int)
 	ints := make(map[int64]int)
 	strings := make(map[string]int)
 	floats := make(map[float64]int)
@@ -106,9 +107,14 @@ func (b *Bytecode) RemoveDuplicates() {
 	for curIdx, c := range b.Constants {
 		switch c := c.(type) {
 		case *CompiledFunction:
-			// add to deduped list
-			indexMap[curIdx] = len(deduped)
-			deduped = append(deduped, c)
+			if newIdx, ok := fns[c]; ok {
+				indexMap[curIdx] = newIdx
+			} else {
+				newIdx = len(deduped)
+				fns[c] = newIdx
+				indexMap[curIdx] = newIdx
+				deduped = append(deduped, c)
+			}
 		case *ImmutableMap:
 			modName := inferModuleName(c)
 			newIdx, ok := immutableMaps[modName]

vendor/github.com/d5/tengo/v2/compiler.go

@@ -44,6 +44,7 @@ type Compiler struct {
 	file            *parser.SourceFile
 	parent          *Compiler
 	modulePath      string
+	importDir       string
 	constants       []Object
 	symbolTable     *SymbolTable
 	scopes          []compilationScope
@@ -505,7 +506,11 @@ func (c *Compiler) Compile(node parser.Node) error {
 				return err
 			}
 		}
-		c.emit(node, parser.OpCall, len(node.Args))
+		ellipsis := 0
+		if node.Ellipsis.IsValid() {
+			ellipsis = 1
+		}
+		c.emit(node, parser.OpCall, len(node.Args), ellipsis)
 	case *parser.ImportExpr:
 		if node.ModuleName == "" {
 			return c.errorf(node, "empty module name")
@@ -520,12 +525,12 @@ func (c *Compiler) Compile(node parser.Node) error {
 			switch v := v.(type) {
 			case []byte: // module written in Tengo
 				compiled, err := c.compileModule(node,
-					node.ModuleName, node.ModuleName, v)
+					node.ModuleName, v, false)
 				if err != nil {
 					return err
 				}
 				c.emit(node, parser.OpConstant, c.addConstant(compiled))
-				c.emit(node, parser.OpCall, 0)
+				c.emit(node, parser.OpCall, 0, 0)
 			case Object: // builtin module
 				c.emit(node, parser.OpConstant, c.addConstant(v))
 			default:
@@ -537,29 +542,25 @@ func (c *Compiler) Compile(node parser.Node) error {
 				moduleName += ".tengo"
 			}
 
-			modulePath, err := filepath.Abs(moduleName)
+			modulePath, err := filepath.Abs(
+				filepath.Join(c.importDir, moduleName))
 			if err != nil {
 				return c.errorf(node, "module file path error: %s",
 					err.Error())
 			}
 
-			if err := c.checkCyclicImports(node, modulePath); err != nil {
-				return err
-			}
-
-			moduleSrc, err := ioutil.ReadFile(moduleName)
+			moduleSrc, err := ioutil.ReadFile(modulePath)
 			if err != nil {
 				return c.errorf(node, "module file read error: %s",
 					err.Error())
 			}
 
-			compiled, err := c.compileModule(node,
-				moduleName, modulePath, moduleSrc)
+			compiled, err := c.compileModule(node, modulePath, moduleSrc, true)
 			if err != nil {
 				return err
 			}
 			c.emit(node, parser.OpConstant, c.addConstant(compiled))
-			c.emit(node, parser.OpCall, 0)
+			c.emit(node, parser.OpCall, 0, 0)
 		} else {
 			return c.errorf(node, "module '%s' not found", node.ModuleName)
 		}
@@ -634,6 +635,11 @@ func (c *Compiler) EnableFileImport(enable bool) {
 	c.allowFileImport = enable
 }
 
+// SetImportDir sets the initial import directory path for file imports.
+func (c *Compiler) SetImportDir(dir string) {
+	c.importDir = dir
+}
+
 func (c *Compiler) compileAssign(
 	node parser.Node,
 	lhs, rhs []parser.Expr,
@@ -847,8 +853,8 @@ func (c *Compiler) compileForInStmt(stmt *parser.ForInStmt) error {
 	//     ... body ...
 	//   }
 	//
-	// ":it" is a local variable but will be conflict with other user variables
-	// because character ":" is not allowed.
+	// ":it" is a local variable but it will not conflict with other user variables
+	// because character ":" is not allowed in the variable names.
 
 	// init
 	//   :it = iterator(iterable)
@@ -893,6 +899,7 @@ func (c *Compiler) compileForInStmt(stmt *parser.ForInStmt) error {
 		if keySymbol.Scope == ScopeGlobal {
 			c.emit(stmt, parser.OpSetGlobal, keySymbol.Index)
 		} else {
+			keySymbol.LocalAssigned = true
 			c.emit(stmt, parser.OpDefineLocal, keySymbol.Index)
 		}
 	}
@@ -909,6 +916,7 @@ func (c *Compiler) compileForInStmt(stmt *parser.ForInStmt) error {
 		if valueSymbol.Scope == ScopeGlobal {
 			c.emit(stmt, parser.OpSetGlobal, valueSymbol.Index)
 		} else {
+			valueSymbol.LocalAssigned = true
 			c.emit(stmt, parser.OpDefineLocal, valueSymbol.Index)
 		}
 	}
@@ -955,8 +963,9 @@ func (c *Compiler) checkCyclicImports(
 
 func (c *Compiler) compileModule(
 	node parser.Node,
-	moduleName, modulePath string,
+	modulePath string,
 	src []byte,
+	isFile bool,
 ) (*CompiledFunction, error) {
 	if err := c.checkCyclicImports(node, modulePath); err != nil {
 		return nil, err
@@ -967,7 +976,7 @@ func (c *Compiler) compileModule(
 		return compiledModule, nil
 	}
 
-	modFile := c.file.Set().AddFile(moduleName, -1, len(src))
+	modFile := c.file.Set().AddFile(modulePath, -1, len(src))
 	p := parser.NewParser(modFile, src, nil)
 	file, err := p.ParseFile()
 	if err != nil {
@@ -984,7 +993,7 @@ func (c *Compiler) compileModule(
 	symbolTable = symbolTable.Fork(false)
 
 	// compile module
-	moduleCompiler := c.fork(modFile, modulePath, symbolTable)
+	moduleCompiler := c.fork(modFile, modulePath, symbolTable, isFile)
 	if err := moduleCompiler.Compile(file); err != nil {
 		return nil, err
 	}
@@ -1082,10 +1091,16 @@ func (c *Compiler) fork(
 	file *parser.SourceFile,
 	modulePath string,
 	symbolTable *SymbolTable,
+	isFile bool,
 ) *Compiler {
 	child := NewCompiler(file, symbolTable, nil, c.modules, c.trace)
 	child.modulePath = modulePath // module file path
 	child.parent = c              // parent to set to current compiler
+	child.allowFileImport = c.allowFileImport
+	child.importDir = c.importDir
+	if isFile && c.importDir != "" {
+		child.importDir = filepath.Dir(modulePath)
+	}
 	return child
 }
 
@@ -1192,6 +1207,7 @@ func (c *Compiler) optimizeFunc(node parser.Node) {
 	var lastOp parser.Opcode
 	var appendReturn bool
 	endPos := len(c.scopes[c.scopeIndex].Instructions)
+	newEndPost := len(newInsts)
 	iterateInstructions(newInsts,
 		func(pos int, opcode parser.Opcode, operands []int) bool {
 			switch opcode {
@@ -1204,6 +1220,8 @@ func (c *Compiler) optimizeFunc(node parser.Node) {
 				} else if endPos == operands[0] {
 					// there's a jump instruction that jumps to the end of
 					// function compiler should append "return".
+					copy(newInsts[pos:],
+						MakeInstruction(opcode, newEndPost))
 					appendReturn = true
 				} else {
 					panic(fmt.Errorf("invalid jump position: %d", newDst))

vendor/github.com/d5/tengo/v2/objects.go

@@ -1342,6 +1342,38 @@ func (o *String) BinaryOp(op token.Token, rhs Object) (Object, error) {
 			}
 			return &String{Value: o.Value + rhsStr}, nil
 		}
+	case token.Less:
+		switch rhs := rhs.(type) {
+		case *String:
+			if o.Value < rhs.Value {
+				return TrueValue, nil
+			}
+			return FalseValue, nil
+		}
+	case token.LessEq:
+		switch rhs := rhs.(type) {
+		case *String:
+			if o.Value <= rhs.Value {
+				return TrueValue, nil
+			}
+			return FalseValue, nil
+		}
+	case token.Greater:
+		switch rhs := rhs.(type) {
+		case *String:
+			if o.Value > rhs.Value {
+				return TrueValue, nil
+			}
+			return FalseValue, nil
+		}
+	case token.GreaterEq:
+		switch rhs := rhs.(type) {
+		case *String:
+			if o.Value >= rhs.Value {
+				return TrueValue, nil
+			}
+			return FalseValue, nil
+		}
 	}
 	return nil, ErrInvalidOperator
 }

vendor/github.com/d5/tengo/v2/parser/expr.go

@@ -111,10 +111,11 @@ func (e *BoolLit) String() string {
 
 // CallExpr represents a function call expression.
 type CallExpr struct {
-	Func   Expr
-	LParen Pos
-	Args   []Expr
-	RParen Pos
+	Func     Expr
+	LParen   Pos
+	Args     []Expr
+	Ellipsis Pos
+	RParen   Pos
 }
 
 func (e *CallExpr) exprNode() {}
@@ -134,6 +135,9 @@ func (e *CallExpr) String() string {
 	for _, e := range e.Args {
 		args = append(args, e.String())
 	}
+	if len(args) > 0 && e.Ellipsis.IsValid() {
+		args[len(args)-1] = args[len(args)-1] + "..."
+	}
 	return e.Func.String() + "(" + strings.Join(args, ", ") + ")"
 }
 

vendor/github.com/d5/tengo/v2/parser/opcodes.go

@@ -120,7 +120,7 @@ var OpcodeOperands = [...][]int{
 	OpImmutable:     {},
 	OpIndex:         {},
 	OpSliceIndex:    {},
-	OpCall:          {1},
+	OpCall:          {1, 1},
 	OpReturn:        {1},
 	OpGetLocal:      {1},
 	OpSetLocal:      {1},

vendor/github.com/d5/tengo/v2/parser/parser.go

@@ -270,9 +270,13 @@ func (p *Parser) parseCall(x Expr) *CallExpr {
 	p.exprLevel++
 
 	var list []Expr
-	for p.token != token.RParen && p.token != token.EOF {
+	var ellipsis Pos
+	for p.token != token.RParen && p.token != token.EOF && !ellipsis.IsValid() {
 		list = append(list, p.parseExpr())
-
+		if p.token == token.Ellipsis {
+			ellipsis = p.pos
+			p.next()
+		}
 		if !p.expectComma(token.RParen, "call argument") {
 			break
 		}
@@ -281,10 +285,11 @@ func (p *Parser) parseCall(x Expr) *CallExpr {
 	p.exprLevel--
 	rparen := p.expect(token.RParen)
 	return &CallExpr{
-		Func:   x,
-		LParen: lparen,
-		RParen: rparen,
-		Args:   list,
+		Func:     x,
+		LParen:   lparen,
+		RParen:   rparen,
+		Ellipsis: ellipsis,
+		Args:     list,
 	}
 }
 

vendor/github.com/d5/tengo/v2/script.go

@@ -3,6 +3,7 @@ package tengo
 import (
 	"context"
 	"fmt"
+	"path/filepath"
 	"sync"
 
 	"github.com/d5/tengo/v2/parser"
@@ -16,6 +17,7 @@ type Script struct {
 	maxAllocs        int64
 	maxConstObjects  int
 	enableFileImport bool
+	importDir        string
 }
 
 // NewScript creates a Script instance with an input script.
@@ -56,6 +58,16 @@ func (s *Script) SetImports(modules *ModuleMap) {
 	s.modules = modules
 }
 
+// SetImportDir sets the initial import directory for script files.
+func (s *Script) SetImportDir(dir string) error {
+	dir, err := filepath.Abs(dir)
+	if err != nil {
+		return err
+	}
+	s.importDir = dir
+	return nil
+}
+
 // SetMaxAllocs sets the maximum number of objects allocations during the run
 // time. Compiled script will return ErrObjectAllocLimit error if it
 // exceeds this limit.
@@ -93,6 +105,7 @@ func (s *Script) Compile() (*Compiled, error) {
 
 	c := NewCompiler(srcFile, symbolTable, nil, s.modules, nil)
 	c.EnableFileImport(s.enableFileImport)
+	c.SetImportDir(s.importDir)
 	if err := c.Compile(file); err != nil {
 		return nil, err
 	}

vendor/github.com/d5/tengo/v2/stdlib/json/encode.go

@@ -1,20 +1,129 @@
 // A modified version of Go's JSON implementation.
 
-// Copyright 2010 The Go Authors. All rights reserved.
+// Copyright 2010, 2020 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
 package json
 
 import (
+	"bytes"
 	"encoding/base64"
 	"errors"
 	"math"
 	"strconv"
+	"unicode/utf8"
 
 	"github.com/d5/tengo/v2"
 )
 
+// safeSet holds the value true if the ASCII character with the given array
+// position can be represented inside a JSON string without any further
+// escaping.
+//
+// All values are true except for the ASCII control characters (0-31), the
+// double quote ("), and the backslash character ("\").
+var safeSet = [utf8.RuneSelf]bool{
+	' ':      true,
+	'!':      true,
+	'"':      false,
+	'#':      true,
+	'$':      true,
+	'%':      true,
+	'&':      true,
+	'\'':     true,
+	'(':      true,
+	')':      true,
+	'*':      true,
+	'+':      true,
+	',':      true,
+	'-':      true,
+	'.':      true,
+	'/':      true,
+	'0':      true,
+	'1':      true,
+	'2':      true,
+	'3':      true,
+	'4':      true,
+	'5':      true,
+	'6':      true,
+	'7':      true,
+	'8':      true,
+	'9':      true,
+	':':      true,
+	';':      true,
+	'<':      true,
+	'=':      true,
+	'>':      true,
+	'?':      true,
+	'@':      true,
+	'A':      true,
+	'B':      true,
+	'C':      true,
+	'D':      true,
+	'E':      true,
+	'F':      true,
+	'G':      true,
+	'H':      true,
+	'I':      true,
+	'J':      true,
+	'K':      true,
+	'L':      true,
+	'M':      true,
+	'N':      true,
+	'O':      true,
+	'P':      true,
+	'Q':      true,
+	'R':      true,
+	'S':      true,
+	'T':      true,
+	'U':      true,
+	'V':      true,
+	'W':      true,
+	'X':      true,
+	'Y':      true,
+	'Z':      true,
+	'[':      true,
+	'\\':     false,
+	']':      true,
+	'^':      true,
+	'_':      true,
+	'`':      true,
+	'a':      true,
+	'b':      true,
+	'c':      true,
+	'd':      true,
+	'e':      true,
+	'f':      true,
+	'g':      true,
+	'h':      true,
+	'i':      true,
+	'j':      true,
+	'k':      true,
+	'l':      true,
+	'm':      true,
+	'n':      true,
+	'o':      true,
+	'p':      true,
+	'q':      true,
+	'r':      true,
+	's':      true,
+	't':      true,
+	'u':      true,
+	'v':      true,
+	'w':      true,
+	'x':      true,
+	'y':      true,
+	'z':      true,
+	'{':      true,
+	'|':      true,
+	'}':      true,
+	'~':      true,
+	'\u007f': true,
+}
+
+var hex = "0123456789abcdef"
+
 // Encode returns the JSON encoding of the object.
 func Encode(o tengo.Object) ([]byte, error) {
 	var b []byte
@@ -53,7 +162,7 @@ func Encode(o tengo.Object) ([]byte, error) {
 		len1 := len(o.Value) - 1
 		idx := 0
 		for key, value := range o.Value {
-			b = strconv.AppendQuote(b, key)
+			b = encodeString(b, key)
 			b = append(b, ':')
 			eb, err := Encode(value)
 			if err != nil {
@@ -71,7 +180,7 @@ func Encode(o tengo.Object) ([]byte, error) {
 		len1 := len(o.Value) - 1
 		idx := 0
 		for key, value := range o.Value {
-			b = strconv.AppendQuote(b, key)
+			b = encodeString(b, key)
 			b = append(b, ':')
 			eb, err := Encode(value)
 			if err != nil {
@@ -130,7 +239,9 @@ func Encode(o tengo.Object) ([]byte, error) {
 	case *tengo.Int:
 		b = strconv.AppendInt(b, o.Value, 10)
 	case *tengo.String:
-		b = strconv.AppendQuote(b, o.Value)
+		// string encoding bug is fixed with newly introduced function
+		// encodeString(). See: https://github.com/d5/tengo/issues/268
+		b = encodeString(b, o.Value)
 	case *tengo.Time:
 		y, err := o.Value.MarshalJSON()
 		if err != nil {
@@ -144,3 +255,79 @@ func Encode(o tengo.Object) ([]byte, error) {
 	}
 	return b, nil
 }
+
+// encodeString encodes given string as JSON string according to
+// https://www.json.org/img/string.png
+// Implementation is inspired by https://github.com/json-iterator/go
+// See encodeStringSlowPath() for more information.
+func encodeString(b []byte, val string) []byte {
+	valLen := len(val)
+	buf := bytes.NewBuffer(b)
+	buf.WriteByte('"')
+
+	// write string, the fast path, without utf8 and escape support
+	i := 0
+	for ; i < valLen; i++ {
+		c := val[i]
+		if c > 31 && c != '"' && c != '\\' {
+			buf.WriteByte(c)
+		} else {
+			break
+		}
+	}
+	if i == valLen {
+		buf.WriteByte('"')
+		return buf.Bytes()
+	}
+	encodeStringSlowPath(buf, i, val, valLen)
+	buf.WriteByte('"')
+	return buf.Bytes()
+}
+
+// encodeStringSlowPath is ported from Go 1.14.2 encoding/json package.
+// U+2028 U+2029 JSONP security holes can be fixed with addition call to
+// json.html_escape() thus it is removed from the implementation below.
+// Note: Invalid runes are not checked as they are checked in original
+// implementation.
+func encodeStringSlowPath(buf *bytes.Buffer, i int, val string, valLen int) {
+	start := i
+	for i < valLen {
+		if b := val[i]; b < utf8.RuneSelf {
+			if safeSet[b] {
+				i++
+				continue
+			}
+			if start < i {
+				buf.WriteString(val[start:i])
+			}
+			buf.WriteByte('\\')
+			switch b {
+			case '\\', '"':
+				buf.WriteByte(b)
+			case '\n':
+				buf.WriteByte('n')
+			case '\r':
+				buf.WriteByte('r')
+			case '\t':
+				buf.WriteByte('t')
+			default:
+				// This encodes bytes < 0x20 except for \t, \n and \r.
+				// If escapeHTML is set, it also escapes <, >, and &
+				// because they can lead to security holes when
+				// user-controlled strings are rendered into JSON
+				// and served to some browsers.
+				buf.WriteString(`u00`)
+				buf.WriteByte(hex[b>>4])
+				buf.WriteByte(hex[b&0xF])
+			}
+			i++
+			start = i
+			continue
+		}
+		i++
+		continue
+	}
+	if start < valLen {
+		buf.WriteString(val[start:])
+	}
+}

vendor/github.com/d5/tengo/v2/vm.go

@@ -80,14 +80,14 @@ func (v *VM) Run() (err error) {
 	if err != nil {
 		filePos := v.fileSet.Position(
 			v.curFrame.fn.SourcePos(v.ip - 1))
-		err = fmt.Errorf("Runtime Error: %s\n\tat %s",
-			err.Error(), filePos)
+		err = fmt.Errorf("Runtime Error: %w\n\tat %s",
+			err, filePos)
 		for v.framesIndex > 1 {
 			v.framesIndex--
 			v.curFrame = &v.frames[v.framesIndex-1]
 			filePos = v.fileSet.Position(
 				v.curFrame.fn.SourcePos(v.curFrame.ip - 1))
-			err = fmt.Errorf("%s\n\tat %s", err.Error(), filePos)
+			err = fmt.Errorf("%w\n\tat %s", err, filePos)
 		}
 		return err
 	}
@@ -537,12 +537,36 @@ func (v *VM) run() {
 			}
 		case parser.OpCall:
 			numArgs := int(v.curInsts[v.ip+1])
-			v.ip++
+			spread := int(v.curInsts[v.ip+2])
+			v.ip += 2
+
 			value := v.stack[v.sp-1-numArgs]
 			if !value.CanCall() {
 				v.err = fmt.Errorf("not callable: %s", value.TypeName())
 				return
 			}
+
+			if spread == 1 {
+				v.sp--
+				switch arr := v.stack[v.sp].(type) {
+				case *Array:
+					for _, item := range arr.Value {
+						v.stack[v.sp] = item
+						v.sp++
+					}
+					numArgs += len(arr.Value) - 1
+				case *ImmutableArray:
+					for _, item := range arr.Value {
+						v.stack[v.sp] = item
+						v.sp++
+					}
+					numArgs += len(arr.Value) - 1
+				default:
+					v.err = fmt.Errorf("not an array: %s", arr.TypeName())
+					return
+				}
+			}
+
 			if callee, ok := value.(*CompiledFunction); ok {
 				if callee.VarArgs {
 					// if the closure is variadic,

vendor/github.com/dyatlov/go-opengraph/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Vitaly Dyatlov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

vendor/github.com/dyatlov/go-opengraph/opengraph/opengraph.go

@@ -0,0 +1,365 @@
+package opengraph
+
+import (
+	"encoding/json"
+	"io"
+	"strconv"
+	"time"
+
+	"golang.org/x/net/html"
+	"golang.org/x/net/html/atom"
+)
+
+// Image defines Open Graph Image type
+type Image struct {
+	URL       string `json:"url"`
+	SecureURL string `json:"secure_url"`
+	Type      string `json:"type"`
+	Width     uint64 `json:"width"`
+	Height    uint64 `json:"height"`
+	draft     bool    `json:"-"`
+}
+
+// Video defines Open Graph Video type
+type Video struct {
+	URL       string `json:"url"`
+	SecureURL string `json:"secure_url"`
+	Type      string `json:"type"`
+	Width     uint64 `json:"width"`
+	Height    uint64 `json:"height"`
+	draft     bool    `json:"-"`
+}
+
+// Audio defines Open Graph Audio Type
+type Audio struct {
+	URL       string `json:"url"`
+	SecureURL string `json:"secure_url"`
+	Type      string `json:"type"`
+	draft     bool    `json:"-"`
+}
+
+// Article contain Open Graph Article structure
+type Article struct {
+	PublishedTime  *time.Time `json:"published_time"`
+	ModifiedTime   *time.Time `json:"modified_time"`
+	ExpirationTime *time.Time `json:"expiration_time"`
+	Section        string     `json:"section"`
+	Tags           []string   `json:"tags"`
+	Authors        []*Profile `json:"authors"`
+}
+
+// Profile contains Open Graph Profile structure
+type Profile struct {
+	FirstName string `json:"first_name"`
+	LastName  string `json:"last_name"`
+	Username  string `json:"username"`
+	Gender    string `json:"gender"`
+}
+
+// Book contains Open Graph Book structure
+type Book struct {
+	ISBN        string     `json:"isbn"`
+	ReleaseDate *time.Time `json:"release_date"`
+	Tags        []string   `json:"tags"`
+	Authors     []*Profile `json:"authors"`
+}
+
+// OpenGraph contains facebook og data
+type OpenGraph struct {
+	isArticle        bool
+	isBook           bool
+	isProfile        bool
+	Type             string   `json:"type"`
+	URL              string   `json:"url"`
+	Title            string   `json:"title"`
+	Description      string   `json:"description"`
+	Determiner       string   `json:"determiner"`
+	SiteName         string   `json:"site_name"`
+	Locale           string   `json:"locale"`
+	LocalesAlternate []string `json:"locales_alternate"`
+	Images           []*Image `json:"images"`
+	Audios           []*Audio `json:"audios"`
+	Videos           []*Video `json:"videos"`
+	Article          *Article `json:"article,omitempty"`
+	Book             *Book    `json:"book,omitempty"`
+	Profile          *Profile `json:"profile,omitempty"`
+}
+
+// NewOpenGraph returns new instance of Open Graph structure
+func NewOpenGraph() *OpenGraph {
+	return &OpenGraph{}
+}
+
+// ToJSON a simple wrapper around json.Marshal
+func (og *OpenGraph) ToJSON() ([]byte, error) {
+	return json.Marshal(og)
+}
+
+// String return json representation of structure, or error string
+func (og *OpenGraph) String() string {
+	data, err := og.ToJSON()
+
+	if err != nil {
+		return err.Error()
+	}
+
+	return string(data[:])
+}
+
+// ProcessHTML parses given html from Reader interface and fills up OpenGraph structure
+func (og *OpenGraph) ProcessHTML(buffer io.Reader) error {
+	z := html.NewTokenizer(buffer)
+	for {
+		tt := z.Next()
+		switch tt {
+		case html.ErrorToken:
+			if z.Err() == io.EOF {
+				return nil
+			}
+			return z.Err()
+		case html.StartTagToken, html.SelfClosingTagToken, html.EndTagToken:
+			name, hasAttr := z.TagName()
+			if atom.Lookup(name) == atom.Body {
+				return nil // OpenGraph is only in head, so we don't need body
+			}
+			if atom.Lookup(name) != atom.Meta || !hasAttr {
+				continue
+			}
+			m := make(map[string]string)
+			var key, val []byte
+			for hasAttr {
+				key, val, hasAttr = z.TagAttr()
+				m[atom.String(key)] = string(val)
+			}
+			og.ProcessMeta(m)
+		}
+	}
+}
+
+func (og *OpenGraph) ensureHasVideo() {
+	if len(og.Videos) > 0 {
+		return
+	}
+	og.Videos = append(og.Videos, &Video{draft: true})
+}
+
+func (og *OpenGraph) ensureHasImage() {
+	if len(og.Images) > 0 {
+		return
+	}
+	og.Images = append(og.Images, &Image{draft: true})
+}
+
+func (og *OpenGraph) ensureHasAudio() {
+	if len(og.Audios) > 0 {
+		return
+	}
+	og.Audios = append(og.Audios, &Audio{draft: true})
+}
+
+// ProcessMeta processes meta attributes and adds them to Open Graph structure if they are suitable for that
+func (og *OpenGraph) ProcessMeta(metaAttrs map[string]string) {
+	switch metaAttrs["property"] {
+	case "og:description":
+		og.Description = metaAttrs["content"]
+	case "og:type":
+		og.Type = metaAttrs["content"]
+		switch og.Type {
+		case "article":
+			og.isArticle = true
+		case "book":
+			og.isBook = true
+		case "profile":
+			og.isProfile = true
+		}
+	case "og:title":
+		og.Title = metaAttrs["content"]
+	case "og:url":
+		og.URL = metaAttrs["content"]
+	case "og:determiner":
+		og.Determiner = metaAttrs["content"]
+	case "og:site_name":
+		og.SiteName = metaAttrs["content"]
+	case "og:locale":
+		og.Locale = metaAttrs["content"]
+	case "og:locale:alternate":
+		og.LocalesAlternate = append(og.LocalesAlternate, metaAttrs["content"])
+	case "og:audio":
+		if len(og.Audios)>0 && og.Audios[len(og.Audios)-1].draft {
+			og.Audios[len(og.Audios)-1].URL = metaAttrs["content"]
+			og.Audios[len(og.Audios)-1].draft = false
+		} else {
+			og.Audios = append(og.Audios, &Audio{URL: metaAttrs["content"]})
+		}
+	case "og:audio:secure_url":
+		og.ensureHasAudio()
+		og.Audios[len(og.Audios)-1].SecureURL = metaAttrs["content"]
+	case "og:audio:type":
+		og.ensureHasAudio()
+		og.Audios[len(og.Audios)-1].Type = metaAttrs["content"]
+	case "og:image":
+		if len(og.Images)>0 && og.Images[len(og.Images)-1].draft {
+			og.Images[len(og.Images)-1].URL = metaAttrs["content"]
+			og.Images[len(og.Images)-1].draft = false
+		} else {
+			og.Images = append(og.Images, &Image{URL: metaAttrs["content"]})
+		}
+	case "og:image:url":
+		og.ensureHasImage()
+		og.Images[len(og.Images)-1].URL = metaAttrs["content"]
+	case "og:image:secure_url":
+		og.ensureHasImage()
+		og.Images[len(og.Images)-1].SecureURL = metaAttrs["content"]
+	case "og:image:type":
+		og.ensureHasImage()
+		og.Images[len(og.Images)-1].Type = metaAttrs["content"]
+	case "og:image:width":
+		w, err := strconv.ParseUint(metaAttrs["content"], 10, 64)
+		if err == nil {
+			og.ensureHasImage()
+			og.Images[len(og.Images)-1].Width = w
+		}
+	case "og:image:height":
+		h, err := strconv.ParseUint(metaAttrs["content"], 10, 64)
+		if err == nil {
+			og.ensureHasImage()
+			og.Images[len(og.Images)-1].Height = h
+		}
+	case "og:video":
+		if len(og.Videos)>0 && og.Videos[len(og.Videos)-1].draft {
+			og.Videos[len(og.Videos)-1].URL = metaAttrs["content"]
+			og.Videos[len(og.Videos)-1].draft = false
+		} else {
+			og.Videos = append(og.Videos, &Video{URL: metaAttrs["content"]})
+		}
+	case "og:video:url":
+		og.ensureHasVideo()
+		og.Videos[len(og.Videos)-1].URL = metaAttrs["content"]
+	case "og:video:secure_url":
+		og.ensureHasVideo()
+		og.Videos[len(og.Videos)-1].SecureURL = metaAttrs["content"]
+	case "og:video:type":
+		og.ensureHasVideo()
+		og.Videos[len(og.Videos)-1].Type = metaAttrs["content"]
+	case "og:video:width":
+		w, err := strconv.ParseUint(metaAttrs["content"], 10, 64)
+		if err == nil {
+			og.ensureHasVideo()
+			og.Videos[len(og.Videos)-1].Width = w
+		}
+	case "og:video:height":
+		h, err := strconv.ParseUint(metaAttrs["content"], 10, 64)
+		if err == nil {
+			og.ensureHasVideo()
+			og.Videos[len(og.Videos)-1].Height = h
+		}
+	default:
+		if og.isArticle {
+			og.processArticleMeta(metaAttrs)
+		} else if og.isBook {
+			og.processBookMeta(metaAttrs)
+		} else if og.isProfile {
+			og.processProfileMeta(metaAttrs)
+		}
+	}
+}
+
+func (og *OpenGraph) processArticleMeta(metaAttrs map[string]string) {
+	if og.Article == nil {
+		og.Article = &Article{}
+	}
+	switch metaAttrs["property"] {
+	case "article:published_time":
+		t, err := time.Parse(time.RFC3339, metaAttrs["content"])
+		if err == nil {
+			og.Article.PublishedTime = &t
+		}
+	case "article:modified_time":
+		t, err := time.Parse(time.RFC3339, metaAttrs["content"])
+		if err == nil {
+			og.Article.ModifiedTime = &t
+		}
+	case "article:expiration_time":
+		t, err := time.Parse(time.RFC3339, metaAttrs["content"])
+		if err == nil {
+			og.Article.ExpirationTime = &t
+		}
+	case "article:section":
+		og.Article.Section = metaAttrs["content"]
+	case "article:tag":
+		og.Article.Tags = append(og.Article.Tags, metaAttrs["content"])
+	case "article:author:first_name":
+		if len(og.Article.Authors) == 0 {
+			og.Article.Authors = append(og.Article.Authors, &Profile{})
+		}
+		og.Article.Authors[len(og.Article.Authors)-1].FirstName = metaAttrs["content"]
+	case "article:author:last_name":
+		if len(og.Article.Authors) == 0 {
+			og.Article.Authors = append(og.Article.Authors, &Profile{})
+		}
+		og.Article.Authors[len(og.Article.Authors)-1].LastName = metaAttrs["content"]
+	case "article:author:username":
+		if len(og.Article.Authors) == 0 {
+			og.Article.Authors = append(og.Article.Authors, &Profile{})
+		}
+		og.Article.Authors[len(og.Article.Authors)-1].Username = metaAttrs["content"]
+	case "article:author:gender":
+		if len(og.Article.Authors) == 0 {
+			og.Article.Authors = append(og.Article.Authors, &Profile{})
+		}
+		og.Article.Authors[len(og.Article.Authors)-1].Gender = metaAttrs["content"]
+	}
+}
+
+func (og *OpenGraph) processBookMeta(metaAttrs map[string]string) {
+	if og.Book == nil {
+		og.Book = &Book{}
+	}
+	switch metaAttrs["property"] {
+	case "book:release_date":
+		t, err := time.Parse(time.RFC3339, metaAttrs["content"])
+		if err == nil {
+			og.Book.ReleaseDate = &t
+		}
+	case "book:isbn":
+		og.Book.ISBN = metaAttrs["content"]
+	case "book:tag":
+		og.Book.Tags = append(og.Book.Tags, metaAttrs["content"])
+	case "book:author:first_name":
+		if len(og.Book.Authors) == 0 {
+			og.Book.Authors = append(og.Book.Authors, &Profile{})
+		}
+		og.Book.Authors[len(og.Book.Authors)-1].FirstName = metaAttrs["content"]
+	case "book:author:last_name":
+		if len(og.Book.Authors) == 0 {
+			og.Book.Authors = append(og.Book.Authors, &Profile{})
+		}
+		og.Book.Authors[len(og.Book.Authors)-1].LastName = metaAttrs["content"]
+	case "book:author:username":
+		if len(og.Book.Authors) == 0 {
+			og.Book.Authors = append(og.Book.Authors, &Profile{})
+		}
+		og.Book.Authors[len(og.Book.Authors)-1].Username = metaAttrs["content"]
+	case "book:author:gender":
+		if len(og.Book.Authors) == 0 {
+			og.Book.Authors = append(og.Book.Authors, &Profile{})
+		}
+		og.Book.Authors[len(og.Book.Authors)-1].Gender = metaAttrs["content"]
+	}
+}
+
+func (og *OpenGraph) processProfileMeta(metaAttrs map[string]string) {
+	if og.Profile == nil {
+		og.Profile = &Profile{}
+	}
+	switch metaAttrs["property"] {
+	case "profile:first_name":
+		og.Profile.FirstName = metaAttrs["content"]
+	case "profile:last_name":
+		og.Profile.LastName = metaAttrs["content"]
+	case "profile:username":
+		og.Profile.Username = metaAttrs["content"]
+	case "profile:gender":
+		og.Profile.Gender = metaAttrs["content"]
+	}
+}

vendor/github.com/francoispqt/gojay/.gitignore

@@ -0,0 +1,5 @@
+vendor
+*.out
+*.log
+*.test
+.vscode

vendor/github.com/francoispqt/gojay/.travis.yml

@@ -0,0 +1,15 @@
+language: go
+
+go:
+  - "1.10.x"
+  - "1.11.x"
+  - "1.12.x"
+
+script:
+  - go get github.com/golang/dep/cmd/dep github.com/stretchr/testify
+  - dep ensure -v -vendor-only
+  - go test ./gojay/codegen/test/... -race
+  - go test -race -coverprofile=coverage.txt -covermode=atomic
+
+after_success:
+  - bash <(curl -s https://codecov.io/bash)

vendor/github.com/francoispqt/gojay/Gopkg.lock

@@ -0,0 +1,163 @@
+# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
+
+
+[[projects]]
+  digest = "1:1a37f9f2ae10d161d9688fb6008ffa14e1631e5068cc3e9698008b9e8d40d575"
+  name = "cloud.google.com/go"
+  packages = ["compute/metadata"]
+  pruneopts = ""
+  revision = "457ea5c15ccf3b87db582c450e80101989da35f7"
+  version = "v0.40.0"
+
+[[projects]]
+  digest = "1:968d8903d598e3fae738325d3410f33f07ea6a2b9ee5591e9c262ee37df6845a"
+  name = "github.com/go-errors/errors"
+  packages = ["."]
+  pruneopts = ""
+  revision = "a6af135bd4e28680facf08a3d206b454abc877a4"
+  version = "v1.0.1"
+
+[[projects]]
+  digest = "1:529d738b7976c3848cae5cf3a8036440166835e389c1f617af701eeb12a0518d"
+  name = "github.com/golang/protobuf"
+  packages = ["proto"]
+  pruneopts = ""
+  revision = "b5d812f8a3706043e23a9cd5babf2e5423744d30"
+  version = "v1.3.1"
+
+[[projects]]
+  branch = "master"
+  digest = "1:cae59d7b8243c671c9f544965522ba35c0fec48ee80adb9f1400cd2f33abbbec"
+  name = "github.com/mailru/easyjson"
+  packages = [
+    ".",
+    "buffer",
+    "jlexer",
+    "jwriter",
+  ]
+  pruneopts = ""
+  revision = "1ea4449da9834f4d333f1cc461c374aea217d249"
+
+[[projects]]
+  digest = "1:1d7e1867c49a6dd9856598ef7c3123604ea3daabf5b83f303ff457bcbc410b1d"
+  name = "github.com/pkg/errors"
+  packages = ["."]
+  pruneopts = ""
+  revision = "ba968bfe8b2f7e042a574c888954fccecfa385b4"
+  version = "v0.8.1"
+
+[[projects]]
+  digest = "1:8d4bbd8ab012efc77ab6b97286f2aff262bcdeac9803bb57d75cf7d0a5e6a877"
+  name = "github.com/viant/assertly"
+  packages = ["."]
+  pruneopts = ""
+  revision = "04f45e0aeb6f3455884877b047a97bcc95dc9493"
+  version = "v0.4.8"
+
+[[projects]]
+  digest = "1:5913451bc2d274673c0716efe226a137625740cd9380641f4d8300ff4f2d82a0"
+  name = "github.com/viant/toolbox"
+  packages = [
+    ".",
+    "cred",
+    "data",
+    "storage",
+    "url",
+  ]
+  pruneopts = ""
+  revision = "1be8e4d172138324f40d55ea61a2aeab0c5ce864"
+  version = "v0.24.0"
+
+[[projects]]
+  branch = "master"
+  digest = "1:9d150270ca2c3356f2224a0878daa1652e4d0b25b345f18b4f6e156cc4b8ec5e"
+  name = "golang.org/x/crypto"
+  packages = [
+    "blowfish",
+    "curve25519",
+    "ed25519",
+    "ed25519/internal/edwards25519",
+    "internal/chacha20",
+    "internal/subtle",
+    "poly1305",
+    "ssh",
+  ]
+  pruneopts = ""
+  revision = "f99c8df09eb5bff426315721bfa5f16a99cad32c"
+
+[[projects]]
+  branch = "master"
+  digest = "1:5a56f211e7c12a65c5585c629457a2fb91d8719844ee8fab92727ea8adb5721c"
+  name = "golang.org/x/net"
+  packages = [
+    "context",
+    "context/ctxhttp",
+    "websocket",
+  ]
+  pruneopts = ""
+  revision = "461777fb6f67e8cb9d70cda16573678d085a74cf"
+
+[[projects]]
+  branch = "master"
+  digest = "1:01bdbbc604dcd5afb6f66a717f69ad45e9643c72d5bc11678d44ffa5c50f9e42"
+  name = "golang.org/x/oauth2"
+  packages = [
+    ".",
+    "google",
+    "internal",
+    "jws",
+    "jwt",
+  ]
+  pruneopts = ""
+  revision = "0f29369cfe4552d0e4bcddc57cc75f4d7e672a33"
+
+[[projects]]
+  branch = "master"
+  digest = "1:8ddb956f67d4c176abbbc42b7514aaeaf9ea30daa24e27d2cf30ad82f9334a2c"
+  name = "golang.org/x/sys"
+  packages = ["cpu"]
+  pruneopts = ""
+  revision = "1e42afee0f762ed3d76e6dd942e4181855fd1849"
+
+[[projects]]
+  digest = "1:47f391ee443f578f01168347818cb234ed819521e49e4d2c8dd2fb80d48ee41a"
+  name = "google.golang.org/appengine"
+  packages = [
+    ".",
+    "internal",
+    "internal/app_identity",
+    "internal/base",
+    "internal/datastore",
+    "internal/log",
+    "internal/modules",
+    "internal/remote_api",
+    "internal/urlfetch",
+    "urlfetch",
+  ]
+  pruneopts = ""
+  revision = "b2f4a3cf3c67576a2ee09e1fe62656a5086ce880"
+  version = "v1.6.1"
+
+[[projects]]
+  digest = "1:cedccf16b71e86db87a24f8d4c70b0a855872eb967cb906a66b95de56aefbd0d"
+  name = "gopkg.in/yaml.v2"
+  packages = ["."]
+  pruneopts = ""
+  revision = "51d6538a90f86fe93ac480b35f37b2be17fef232"
+  version = "v2.2.2"
+
+[solve-meta]
+  analyzer-name = "dep"
+  analyzer-version = 1
+  input-imports = [
+    "github.com/go-errors/errors",
+    "github.com/mailru/easyjson",
+    "github.com/mailru/easyjson/jlexer",
+    "github.com/mailru/easyjson/jwriter",
+    "github.com/viant/assertly",
+    "github.com/viant/toolbox",
+    "github.com/viant/toolbox/url",
+    "golang.org/x/net/websocket",
+  ]
+  solver-name = "gps-cdcl"
+  solver-version = 1

vendor/github.com/francoispqt/gojay/Gopkg.toml

@@ -0,0 +1,23 @@
+# Gopkg.toml example
+#
+# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md
+# for detailed Gopkg.toml documentation.
+#
+# required = ["github.com/user/thing/cmd/thing"]
+# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
+#
+# [[constraint]]
+#   name = "github.com/user/project"
+#   version = "1.0.0"
+#
+# [[constraint]]
+#   name = "github.com/user/project2"
+#   branch = "dev"
+#   source = "github.com/myfork/project2"
+#
+# [[override]]
+#  name = "github.com/x/y"
+#  version = "2.4.0"
+
+
+ignored = ["github.com/francoispqt/benchmarks*","github.com/stretchr/testify*","github.com/stretchr/testify","github.com/json-iterator/go","github.com/buger/jsonparser"]

vendor/github.com/francoispqt/gojay/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2016 gojay
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

vendor/github.com/francoispqt/gojay/Makefile

@@ -0,0 +1,11 @@
+.PHONY: test
+test:
+	go test -race -run=^Test -v
+
+.PHONY: cover
+cover: 
+	go test -coverprofile=coverage.out -covermode=atomic
+
+.PHONY: coverhtml
+coverhtml: 
+	go tool cover -html=coverage.out

vendor/github.com/francoispqt/gojay/README.md

@@ -0,0 +1,855 @@
+[![Build Status](https://travis-ci.org/francoispqt/gojay.svg?branch=master)](https://travis-ci.org/francoispqt/gojay)
+[![codecov](https://codecov.io/gh/francoispqt/gojay/branch/master/graph/badge.svg)](https://codecov.io/gh/francoispqt/gojay)
+[![Go Report Card](https://goreportcard.com/badge/github.com/francoispqt/gojay)](https://goreportcard.com/report/github.com/francoispqt/gojay)
+[![Go doc](http://img.shields.io/badge/go-documentation-blue.svg?style=flat-square
+)](https://godoc.org/github.com/francoispqt/gojay)
+![MIT License](https://img.shields.io/badge/license-mit-blue.svg?style=flat-square)
+[![Sourcegraph](https://sourcegraph.com/github.com/francoispqt/gojay/-/badge.svg)](https://sourcegraph.com/github.com/francoispqt/gojay)
+![stability-stable](https://img.shields.io/badge/stability-stable-green.svg)
+
+# GoJay
+
+<img src="https://github.com/francoispqt/gojay/raw/master/gojay.png" width="200px">
+
+GoJay is a performant JSON encoder/decoder for Golang (currently the most performant, [see benchmarks](#benchmark-results)).
+
+It has a simple API and doesn't use reflection. It relies on small interfaces to decode/encode structures and slices.
+
+Gojay also comes with powerful stream decoding features and an even faster [Unsafe](#unsafe-api) API.
+
+There is also a [code generation tool](https://github.com/francoispqt/gojay/tree/master/gojay) to make usage easier and faster.
+
+# Why another JSON parser?
+
+I looked at other fast decoder/encoder and realised it was mostly hardly readable static code generation or a lot of reflection, poor streaming features, and not so fast in the end.
+
+Also, I wanted to build a decoder that could consume an io.Reader of line or comma delimited JSON, in a JIT way. To consume a flow of JSON objects from a TCP connection for example or from a standard output. Same way I wanted to build an encoder that could encode a flow of data to a io.Writer.
+
+This is how GoJay aims to be a very fast, JIT stream parser with 0 reflection, low allocation with a friendly API.
+
+# Get started
+
+```bash
+go get github.com/francoispqt/gojay
+```
+
+* [Encoder](#encoding)
+* [Decoder](#decoding)
+* [Stream API](#stream-api)
+* [Code Generation](https://github.com/francoispqt/gojay/tree/master/gojay)
+
+## Decoding
+
+Decoding is done through two different API similar to standard `encoding/json`:
+* [Unmarshal](#unmarshal-api)
+* [Decode](#decode-api)
+
+
+Example of basic stucture decoding with Unmarshal:
+```go
+import "github.com/francoispqt/gojay"
+
+type user struct {
+    id int
+    name string
+    email string
+}
+// implement gojay.UnmarshalerJSONObject
+func (u *user) UnmarshalJSONObject(dec *gojay.Decoder, key string) error {
+    switch key {
+    case "id":
+        return dec.Int(&u.id)
+    case "name":
+        return dec.String(&u.name)
+    case "email":
+        return dec.String(&u.email)
+    }
+    return nil
+}
+func (u *user) NKeys() int {
+    return 3
+}
+
+func main() {
+    u := &user{}
+    d := []byte(`{"id":1,"name":"gojay","email":"gojay@email.com"}`)
+    err := gojay.UnmarshalJSONObject(d, u)
+    if err != nil {
+        log.Fatal(err)
+    }
+}
+```
+
+with Decode:
+```go
+func main() {
+    u := &user{}
+    dec := gojay.NewDecoder(bytes.NewReader([]byte(`{"id":1,"name":"gojay","email":"gojay@email.com"}`)))
+    err := dec.DecodeObject(d, u)
+    if err != nil {
+        log.Fatal(err)
+    }
+}
+```
+
+### Unmarshal API
+
+Unmarshal API decodes a `[]byte` to a given pointer with a single function.
+
+Behind the doors, Unmarshal API borrows a `*gojay.Decoder` resets its settings and decodes the data to the given pointer and releases the `*gojay.Decoder` to the pool when it finishes, whether it encounters an error or not.
+
+If it cannot find the right Decoding strategy for the type of the given pointer, it returns an `InvalidUnmarshalError`. You can test the error returned by doing `if ok := err.(InvalidUnmarshalError); ok {}`.
+
+Unmarshal API comes with three functions:
+* Unmarshal
+```go
+func Unmarshal(data []byte, v interface{}) error
+```
+
+* UnmarshalJSONObject
+```go
+func UnmarshalJSONObject(data []byte, v gojay.UnmarshalerJSONObject) error
+```
+
+* UnmarshalJSONArray
+```go
+func UnmarshalJSONArray(data []byte, v gojay.UnmarshalerJSONArray) error
+```
+
+
+### Decode API
+
+Decode API decodes a `[]byte` to a given pointer by creating or borrowing a `*gojay.Decoder` with an `io.Reader` and calling `Decode` methods.
+
+__Getting a *gojay.Decoder or Borrowing__
+
+You can either get a fresh `*gojay.Decoder` calling `dec := gojay.NewDecoder(io.Reader)` or borrow one from the pool by calling `dec := gojay.BorrowDecoder(io.Reader)`.
+
+After using a decoder, you can release it by calling `dec.Release()`. Beware, if you reuse the decoder after releasing it, it will panic with an error of type `InvalidUsagePooledDecoderError`. If you want to fully benefit from the pooling, you must release your decoders after using.
+
+Example getting a fresh an releasing:
+```go
+str := ""
+dec := gojay.NewDecoder(strings.NewReader(`"test"`))
+defer dec.Release()
+if err := dec.Decode(&str); err != nil {
+    log.Fatal(err)
+}
+```
+Example borrowing a decoder and releasing:
+```go
+str := ""
+dec := gojay.BorrowDecoder(strings.NewReader(`"test"`))
+defer dec.Release()
+if err := dec.Decode(&str); err != nil {
+    log.Fatal(err)
+}
+```
+
+`*gojay.Decoder` has multiple methods to decode to specific types:
+* Decode
+```go
+func (dec *gojay.Decoder) Decode(v interface{}) error
+```
+* DecodeObject
+```go
+func (dec *gojay.Decoder) DecodeObject(v gojay.UnmarshalerJSONObject) error
+```
+* DecodeArray
+```go
+func (dec *gojay.Decoder) DecodeArray(v gojay.UnmarshalerJSONArray) error
+```
+* DecodeInt
+```go
+func (dec *gojay.Decoder) DecodeInt(v *int) error
+```
+* DecodeBool
+```go
+func (dec *gojay.Decoder) DecodeBool(v *bool) error
+```
+* DecodeString
+```go
+func (dec *gojay.Decoder) DecodeString(v *string) error
+```
+
+All DecodeXxx methods are used to decode top level JSON values. If you are decoding keys or items of a JSON object or array, don't use the Decode methods.
+
+Example:
+```go
+reader := strings.NewReader(`"John Doe"`)
+dec := NewDecoder(reader)
+
+var str string
+err := dec.DecodeString(&str)
+if err != nil {
+    log.Fatal(err)
+}
+
+fmt.Println(str) // John Doe
+```
+
+### Structs and Maps
+#### UnmarshalerJSONObject Interface
+
+To unmarshal a JSON object to a structure, the structure must implement the `UnmarshalerJSONObject` interface:
+```go
+type UnmarshalerJSONObject interface {
+	UnmarshalJSONObject(*gojay.Decoder, string) error
+	NKeys() int
+}
+```
+`UnmarshalJSONObject` method takes two arguments, the first one is a pointer to the Decoder (*gojay.Decoder) and the second one is the string value of the current key being parsed. If the JSON data is not an object, the UnmarshalJSONObject method will never be called.
+
+`NKeys` method must return the number of keys to Unmarshal in the JSON object or 0. If zero is returned, all keys will be parsed.
+
+Example of implementation for a struct:
+```go
+type user struct {
+    id int
+    name string
+    email string
+}
+// implement UnmarshalerJSONObject
+func (u *user) UnmarshalJSONObject(dec *gojay.Decoder, key string) error {
+    switch key {
+    case "id":
+        return dec.Int(&u.id)
+    case "name":
+        return dec.String(&u.name)
+    case "email":
+        return dec.String(&u.email)
+    }
+    return nil
+}
+func (u *user) NKeys() int {
+    return 3
+}
+```
+
+Example of implementation for a `map[string]string`:
+```go
+// define our custom map type implementing UnmarshalerJSONObject
+type message map[string]string
+
+// Implementing Unmarshaler
+func (m message) UnmarshalJSONObject(dec *gojay.Decoder, k string) error {
+	str := ""
+	err := dec.String(&str)
+	if err != nil {
+		return err
+	}
+	m[k] = str
+	return nil
+}
+
+// we return 0, it tells the Decoder to decode all keys
+func (m message) NKeys() int {
+	return 0
+}
+```
+
+### Arrays, Slices and Channels
+
+To unmarshal a JSON object to a slice an array or a channel, it must implement the UnmarshalerJSONArray interface:
+```go
+type UnmarshalerJSONArray interface {
+	UnmarshalJSONArray(*gojay.Decoder) error
+}
+```
+UnmarshalJSONArray method takes one argument, a pointer to the Decoder (*gojay.Decoder). If the JSON data is not an array, the Unmarshal method will never be called.
+
+Example of implementation with a slice:
+```go
+type testSlice []string
+// implement UnmarshalerJSONArray
+func (t *testSlice) UnmarshalJSONArray(dec *gojay.Decoder) error {
+	str := ""
+	if err := dec.String(&str); err != nil {
+		return err
+	}
+	*t = append(*t, str)
+	return nil
+}
+
+func main() {
+	dec := gojay.BorrowDecoder(strings.NewReader(`["Tom", "Jim"]`))
+	var slice testSlice
+	err := dec.DecodeArray(&slice)
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Println(slice) // [Tom Jim]
+	dec.Release()
+}
+```
+
+Example of implementation with a channel:
+```go
+type testChannel chan string
+// implement UnmarshalerJSONArray
+func (c testChannel) UnmarshalJSONArray(dec *gojay.Decoder) error {
+	str := ""
+	if err := dec.String(&str); err != nil {
+		return err
+	}
+	c <- str
+	return nil
+}
+
+func main() {
+	dec := gojay.BorrowDecoder(strings.NewReader(`["Tom", "Jim"]`))
+	c := make(testChannel, 2)
+	err := dec.DecodeArray(c)
+	if err != nil {
+		log.Fatal(err)
+	}
+	for i := 0; i < 2; i++ {
+		fmt.Println(<-c)
+	}
+	close(c)
+	dec.Release()
+}
+```
+
+Example of implementation with an array:
+```go
+type testArray [3]string
+// implement UnmarshalerJSONArray
+func (a *testArray) UnmarshalJSONArray(dec *Decoder) error {
+	var str string
+	if err := dec.String(&str); err != nil {
+		return err
+	}
+	a[dec.Index()] = str
+	return nil
+}
+
+func main() {
+	dec := gojay.BorrowDecoder(strings.NewReader(`["Tom", "Jim", "Bob"]`))
+	var a testArray
+	err := dec.DecodeArray(&a)
+	fmt.Println(a) // [Tom Jim Bob]
+	dec.Release()
+}
+```
+
+### Other types
+To decode other types (string, int, int32, int64, uint32, uint64, float, booleans), you don't need to implement any interface.
+
+Example of encoding strings:
+```go
+func main() {
+    json := []byte(`"Jay"`)
+    var v string
+    err := gojay.Unmarshal(json, &v)
+    if err != nil {
+        log.Fatal(err)
+    }
+    fmt.Println(v) // Jay
+}
+```
+
+### Decode values methods
+When decoding a JSON object of a JSON array using `UnmarshalerJSONObject` or `UnmarshalerJSONArray` interface, the `gojay.Decoder` provides dozens of methods to Decode multiple types.
+
+Non exhaustive list of methods available (to see all methods, check the godoc):
+```go
+dec.Int
+dec.Int8
+dec.Int16
+dec.Int32
+dec.Int64
+dec.Uint8
+dec.Uint16
+dec.Uint32
+dec.Uint64
+dec.String
+dec.Time
+dec.Bool
+dec.SQLNullString
+dec.SQLNullInt64
+```
+
+
+## Encoding
+
+Encoding is done through two different API similar to standard `encoding/json`:
+* [Marshal](#marshal-api)
+* [Encode](#encode-api)
+
+Example of basic structure encoding with Marshal:
+```go
+import "github.com/francoispqt/gojay"
+
+type user struct {
+	id    int
+	name  string
+	email string
+}
+
+// implement MarshalerJSONObject
+func (u *user) MarshalJSONObject(enc *gojay.Encoder) {
+	enc.IntKey("id", u.id)
+	enc.StringKey("name", u.name)
+	enc.StringKey("email", u.email)
+}
+func (u *user) IsNil() bool {
+	return u == nil
+}
+
+func main() {
+	u := &user{1, "gojay", "gojay@email.com"}
+	b, err := gojay.MarshalJSONObject(u)
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Println(string(b)) // {"id":1,"name":"gojay","email":"gojay@email.com"}
+}
+```
+
+with Encode:
+```go
+func main() {
+	u := &user{1, "gojay", "gojay@email.com"}
+	b := strings.Builder{}
+	enc := gojay.NewEncoder(&b)
+	if err := enc.Encode(u); err != nil {
+		log.Fatal(err)
+	}
+	fmt.Println(b.String()) // {"id":1,"name":"gojay","email":"gojay@email.com"}
+}
+```
+
+### Marshal API
+
+Marshal API encodes a value to a JSON `[]byte` with a single function.
+
+Behind the doors, Marshal API borrows a `*gojay.Encoder` resets its settings and encodes the data to an internal byte buffer and releases the `*gojay.Encoder` to the pool when it finishes, whether it encounters an error or not.
+
+If it cannot find the right Encoding strategy for the type of the given value, it returns an `InvalidMarshalError`. You can test the error returned by doing `if ok := err.(InvalidMarshalError); ok {}`.
+
+Marshal API comes with three functions:
+* Marshal
+```go
+func Marshal(v interface{}) ([]byte, error)
+```
+
+* MarshalJSONObject
+```go
+func MarshalJSONObject(v gojay.MarshalerJSONObject) ([]byte, error)
+```
+
+* MarshalJSONArray
+```go
+func MarshalJSONArray(v gojay.MarshalerJSONArray) ([]byte, error)
+```
+
+### Encode API
+
+Encode API decodes a value to JSON by creating or borrowing a `*gojay.Encoder` sending it to an `io.Writer` and calling `Encode` methods.
+
+__Getting a *gojay.Encoder or Borrowing__
+
+You can either get a fresh `*gojay.Encoder` calling `enc := gojay.NewEncoder(io.Writer)` or borrow one from the pool by calling `enc := gojay.BorrowEncoder(io.Writer)`.
+
+After using an encoder, you can release it by calling `enc.Release()`. Beware, if you reuse the encoder after releasing it, it will panic with an error of type `InvalidUsagePooledEncoderError`. If you want to fully benefit from the pooling, you must release your encoders after using.
+
+Example getting a fresh encoder an releasing:
+```go
+str := "test"
+b := strings.Builder{}
+enc := gojay.NewEncoder(&b)
+defer enc.Release()
+if err := enc.Encode(str); err != nil {
+    log.Fatal(err)
+}
+```
+Example borrowing an encoder and releasing:
+```go
+str := "test"
+b := strings.Builder{}
+enc := gojay.BorrowEncoder(b)
+defer enc.Release()
+if err := enc.Encode(str); err != nil {
+    log.Fatal(err)
+}
+```
+
+`*gojay.Encoder` has multiple methods to encoder specific types to JSON:
+* Encode
+```go
+func (enc *gojay.Encoder) Encode(v interface{}) error
+```
+* EncodeObject
+```go
+func (enc *gojay.Encoder) EncodeObject(v gojay.MarshalerJSONObject) error
+```
+* EncodeArray
+```go
+func (enc *gojay.Encoder) EncodeArray(v gojay.MarshalerJSONArray) error
+```
+* EncodeInt
+```go
+func (enc *gojay.Encoder) EncodeInt(n int) error
+```
+* EncodeInt64
+```go
+func (enc *gojay.Encoder) EncodeInt64(n int64) error
+```
+* EncodeFloat
+```go
+func (enc *gojay.Encoder) EncodeFloat(n float64) error
+```
+* EncodeBool
+```go
+func (enc *gojay.Encoder) EncodeBool(v bool) error
+```
+* EncodeString
+```go
+func (enc *gojay.Encoder) EncodeString(s string) error
+```
+
+### Structs and Maps
+
+To encode a structure, the structure must implement the MarshalerJSONObject interface:
+```go
+type MarshalerJSONObject interface {
+	MarshalJSONObject(enc *gojay.Encoder)
+	IsNil() bool
+}
+```
+`MarshalJSONObject` method takes one argument, a pointer to the Encoder (*gojay.Encoder). The method must add all the keys in the JSON Object by calling Decoder's methods.
+
+IsNil method returns a boolean indicating if the interface underlying value is nil or not. It is used to safely ensure that the underlying value is not nil without using Reflection.
+
+Example of implementation for a struct:
+```go
+type user struct {
+	id    int
+	name  string
+	email string
+}
+
+// implement MarshalerJSONObject
+func (u *user) MarshalJSONObject(enc *gojay.Encoder) {
+	enc.IntKey("id", u.id)
+	enc.StringKey("name", u.name)
+	enc.StringKey("email", u.email)
+}
+func (u *user) IsNil() bool {
+	return u == nil
+}
+```
+
+Example of implementation for a `map[string]string`:
+```go
+// define our custom map type implementing MarshalerJSONObject
+type message map[string]string
+
+// Implementing Marshaler
+func (m message) MarshalJSONObject(enc *gojay.Encoder) {
+	for k, v := range m {
+		enc.StringKey(k, v)
+	}
+}
+
+func (m message) IsNil() bool {
+	return m == nil
+}
+```
+
+### Arrays and Slices
+To encode an array or a slice, the slice/array must implement the MarshalerJSONArray interface:
+```go
+type MarshalerJSONArray interface {
+	MarshalJSONArray(enc *gojay.Encoder)
+	IsNil() bool
+}
+```
+`MarshalJSONArray` method takes one argument, a pointer to the Encoder (*gojay.Encoder). The method must add all element in the JSON Array by calling Decoder's methods.
+
+`IsNil` method returns a boolean indicating if the interface underlying value is nil(empty) or not. It is used to safely ensure that the underlying value is not nil without using Reflection and also to in `OmitEmpty` feature.
+
+Example of implementation:
+```go
+type users []*user
+// implement MarshalerJSONArray
+func (u *users) MarshalJSONArray(enc *gojay.Encoder) {
+	for _, e := range u {
+		enc.Object(e)
+	}
+}
+func (u *users) IsNil() bool {
+	return len(u) == 0
+}
+```
+
+### Other types
+To encode other types (string, int, float, booleans), you don't need to implement any interface.
+
+Example of encoding strings:
+```go
+func main() {
+	name := "Jay"
+	b, err := gojay.Marshal(name)
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Println(string(b)) // "Jay"
+}
+```
+
+# Stream API
+
+### Stream Decoding
+GoJay ships with a powerful stream decoder.
+
+It allows to read continuously from an io.Reader stream and do JIT decoding writing unmarshalled JSON to a channel to allow async consuming.
+
+When using the Stream API, the Decoder implements context.Context to provide graceful cancellation.
+
+To decode a stream of JSON, you must call `gojay.Stream.DecodeStream` and pass it a `UnmarshalerStream` implementation.
+
+```go
+type UnmarshalerStream interface {
+	UnmarshalStream(*StreamDecoder) error
+}
+```
+
+Example of implementation of stream reading from a WebSocket connection:
+```go
+// implement UnmarshalerStream
+type ChannelStream chan *user
+
+func (c ChannelStream) UnmarshalStream(dec *gojay.StreamDecoder) error {
+	u := &user{}
+	if err := dec.Object(u); err != nil {
+		return err
+	}
+	c <- u
+	return nil
+}
+
+func main() {
+	// get our websocket connection
+	origin := "http://localhost/"
+	url := "ws://localhost:12345/ws"
+	ws, err := websocket.Dial(url, "", origin)
+	if err != nil {
+		log.Fatal(err)
+	}
+	// create our channel which will receive our objects
+	streamChan := ChannelStream(make(chan *user))
+	// borrow a decoder
+	dec := gojay.Stream.BorrowDecoder(ws)
+	// start decoding, it will block until a JSON message is decoded from the WebSocket
+	// or until Done channel is closed
+	go dec.DecodeStream(streamChan)
+	for {
+		select {
+		case v := <-streamChan:
+			// Got something from my websocket!
+			log.Println(v)
+		case <-dec.Done():
+			log.Println("finished reading from WebSocket")
+			os.Exit(0)
+		}
+	}
+}
+```
+
+### Stream Encoding
+GoJay ships with a powerful stream encoder part of the Stream API.
+
+It allows to write continuously to an io.Writer and do JIT encoding of data fed to a channel to allow async consuming. You can set multiple consumers on the channel to be as performant as possible. Consumers are non blocking and are scheduled individually in their own go routine.
+
+When using the Stream API, the Encoder implements context.Context to provide graceful cancellation.
+
+To encode a stream of data, you must call `EncodeStream` and pass it a `MarshalerStream` implementation.
+
+```go
+type MarshalerStream interface {
+	MarshalStream(enc *gojay.StreamEncoder)
+}
+```
+
+Example of implementation of stream writing to a WebSocket:
+```go
+// Our structure which will be pushed to our stream
+type user struct {
+	id    int
+	name  string
+	email string
+}
+
+func (u *user) MarshalJSONObject(enc *gojay.Encoder) {
+	enc.IntKey("id", u.id)
+	enc.StringKey("name", u.name)
+	enc.StringKey("email", u.email)
+}
+func (u *user) IsNil() bool {
+	return u == nil
+}
+
+// Our MarshalerStream implementation
+type StreamChan chan *user
+
+func (s StreamChan) MarshalStream(enc *gojay.StreamEncoder) {
+	select {
+	case <-enc.Done():
+		return
+	case o := <-s:
+		enc.Object(o)
+	}
+}
+
+// Our main function
+func main() {
+	// get our websocket connection
+	origin := "http://localhost/"
+	url := "ws://localhost:12345/ws"
+	ws, err := websocket.Dial(url, "", origin)
+	if err != nil {
+		log.Fatal(err)
+	}
+	// we borrow an encoder set stdout as the writer,
+	// set the number of consumer to 10
+	// and tell the encoder to separate each encoded element
+	// added to the channel by a new line character
+	enc := gojay.Stream.BorrowEncoder(ws).NConsumer(10).LineDelimited()
+	// instantiate our MarshalerStream
+	s := StreamChan(make(chan *user))
+	// start the stream encoder
+	// will block its goroutine until enc.Cancel(error) is called
+	// or until something is written to the channel
+	go enc.EncodeStream(s)
+	// write to our MarshalerStream
+	for i := 0; i < 1000; i++ {
+		s <- &user{i, "username", "user@email.com"}
+	}
+	// Wait
+	<-enc.Done()
+}
+```
+
+# Unsafe API
+
+Unsafe API has the same functions than the regular API, it only has `Unmarshal API` for now. It is unsafe because it makes assumptions on the quality of the given JSON.
+
+If you are not sure if your JSON is valid, don't use the Unsafe API.
+
+Also, the `Unsafe` API does not copy the buffer when using Unmarshal API, which, in case of string decoding, can lead to data corruption if a byte buffer is reused. Using the `Decode` API makes `Unsafe` API safer as the io.Reader relies on `copy` builtin method and `Decoder` will have its own internal buffer :)
+
+Access the `Unsafe` API this way:
+```go
+gojay.Unsafe.Unmarshal(b, v)
+```
+
+
+# Benchmarks
+
+Benchmarks encode and decode three different data based on size (small, medium, large).
+
+To run benchmark for decoder:
+```bash
+cd $GOPATH/src/github.com/francoispqt/gojay/benchmarks/decoder && make bench
+```
+
+To run benchmark for encoder:
+```bash
+cd $GOPATH/src/github.com/francoispqt/gojay/benchmarks/encoder && make bench
+```
+
+# Benchmark Results
+## Decode
+
+<img src="https://images2.imgbox.com/78/01/49OExcPh_o.png" width="500px">
+
+### Small Payload
+[benchmark code is here](https://github.com/francoispqt/gojay/blob/master/benchmarks/decoder/decoder_bench_small_test.go)
+
+[benchmark data is here](https://github.com/francoispqt/gojay/blob/master/benchmarks/benchmarks_small.go)
+
+|                 | ns/op     | bytes/op     | allocs/op |
+|-----------------|-----------|--------------|-----------|
+| Std Library     | 2547      | 496          | 4         |
+| JsonIter        | 2046      | 312          | 12        |
+| JsonParser      | 1408      | 0            | 0         |
+| EasyJson        | 929       | 240          | 2         |
+| **GoJay**       | **807**   | **256**      | **2**     |
+| **GoJay-unsafe**| **712**   | **112**      | **1**     |
+
+### Medium Payload
+[benchmark code is here](https://github.com/francoispqt/gojay/blob/master/benchmarks/decoder/decoder_bench_medium_test.go)
+
+[benchmark data is here](https://github.com/francoispqt/gojay/blob/master/benchmarks/benchmarks_medium.go)
+
+|                 | ns/op     | bytes/op | allocs/op |
+|-----------------|-----------|----------|-----------|
+| Std Library     | 30148     | 2152     | 496       |
+| JsonIter        | 16309     | 2976     | 80        |
+| JsonParser      | 7793      | 0        | 0         |
+| EasyJson        | 7957      | 232      | 6         |
+| **GoJay**       | **4984**  | **2448** | **8**     |
+| **GoJay-unsafe**| **4809**  | **144**  | **7**     |
+
+### Large Payload
+[benchmark code is here](https://github.com/francoispqt/gojay/blob/master/benchmarks/decoder/decoder_bench_large_test.go)
+
+[benchmark data is here](https://github.com/francoispqt/gojay/blob/master/benchmarks/benchmarks_large.go)
+
+|                 | ns/op     | bytes/op    | allocs/op |
+|-----------------|-----------|-------------|-----------|
+| JsonIter        | 210078    | 41712       | 1136      |
+| EasyJson        | 106626    | 160         | 2         |
+| JsonParser      | 66813     | 0           | 0         |
+| **GoJay**       | **52153** | **31241**   | **77**    |
+| **GoJay-unsafe**| **48277** | **2561**    | **76**    |
+
+## Encode
+
+<img src="https://images2.imgbox.com/e9/cc/pnM8c7Gf_o.png" width="500px">
+
+### Small Struct
+[benchmark code is here](https://github.com/francoispqt/gojay/blob/master/benchmarks/encoder/encoder_bench_small_test.go)
+
+[benchmark data is here](https://github.com/francoispqt/gojay/blob/master/benchmarks/benchmarks_small.go)
+
+|                | ns/op    | bytes/op     | allocs/op |
+|----------------|----------|--------------|-----------|
+| Std Library    | 1280     | 464          | 3         |
+| EasyJson       | 871      | 944          | 6         |
+| JsonIter       | 866      | 272          | 3         |
+| **GoJay**      | **543**  | **112**      | **1**     |
+| **GoJay-func** | **347**  | **0**        | **0**     |
+
+### Medium Struct
+[benchmark code is here](https://github.com/francoispqt/gojay/blob/master/benchmarks/encoder/encoder_bench_medium_test.go)
+
+[benchmark data is here](https://github.com/francoispqt/gojay/blob/master/benchmarks/benchmarks_medium.go)
+
+|             | ns/op    | bytes/op     | allocs/op |
+|-------------|----------|--------------|-----------|
+| Std Library | 5006     | 1496         | 25        |
+| JsonIter    | 2232     | 1544         | 20        |
+| EasyJson    | 1997     | 1544         | 19        |
+| **GoJay**   | **1522** | **312**      | **14**    |
+
+### Large Struct
+[benchmark code is here](https://github.com/francoispqt/gojay/blob/master/benchmarks/encoder/encoder_bench_large_test.go)
+
+[benchmark data is here](https://github.com/francoispqt/gojay/blob/master/benchmarks/benchmarks_large.go)
+
+|             | ns/op     | bytes/op     | allocs/op |
+|-------------|-----------|--------------|-----------|
+| Std Library | 66441     | 20576        | 332       |
+| JsonIter    | 35247     | 20255        | 328       |
+| EasyJson    | 32053     | 15474        | 327       |
+| **GoJay**   | **27847** | **9802**     | **318**   |
+
+# Contributing
+
+Contributions are welcome :)
+
+If you encounter issues please report it in Github and/or send an email at [francois@parquet.ninja](mailto:francois@parquet.ninja)
+

vendor/github.com/francoispqt/gojay/decode.go

@@ -0,0 +1,386 @@
+package gojay
+
+import (
+	"fmt"
+	"io"
+)
+
+// UnmarshalJSONArray parses the JSON-encoded data and stores the result in the value pointed to by v.
+//
+// v must implement UnmarshalerJSONArray.
+//
+// If a JSON value is not appropriate for a given target type, or if a JSON number
+// overflows the target type, UnmarshalJSONArray skips that field and completes the unmarshaling as best it can.
+func UnmarshalJSONArray(data []byte, v UnmarshalerJSONArray) error {
+	dec := borrowDecoder(nil, 0)
+	defer dec.Release()
+	dec.data = make([]byte, len(data))
+	copy(dec.data, data)
+	dec.length = len(data)
+	_, err := dec.decodeArray(v)
+	if err != nil {
+		return err
+	}
+	if dec.err != nil {
+		return dec.err
+	}
+	return nil
+}
+
+// UnmarshalJSONObject parses the JSON-encoded data and stores the result in the value pointed to by v.
+//
+// v must implement UnmarshalerJSONObject.
+//
+// If a JSON value is not appropriate for a given target type, or if a JSON number
+// overflows the target type, UnmarshalJSONObject skips that field and completes the unmarshaling as best it can.
+func UnmarshalJSONObject(data []byte, v UnmarshalerJSONObject) error {
+	dec := borrowDecoder(nil, 0)
+	defer dec.Release()
+	dec.data = make([]byte, len(data))
+	copy(dec.data, data)
+	dec.length = len(data)
+	_, err := dec.decodeObject(v)
+	if err != nil {
+		return err
+	}
+	if dec.err != nil {
+		return dec.err
+	}
+	return nil
+}
+
+// Unmarshal parses the JSON-encoded data and stores the result in the value pointed to by v.
+// If v is nil, not an implementation of UnmarshalerJSONObject or UnmarshalerJSONArray or not one of the following types:
+// 	*string, **string, *int, **int, *int8, **int8, *int16, **int16, *int32, **int32, *int64, **int64, *uint8, **uint8, *uint16, **uint16,
+// 	*uint32, **uint32, *uint64, **uint64, *float64, **float64, *float32, **float32, *bool, **bool
+// Unmarshal returns an InvalidUnmarshalError.
+//
+//
+// If a JSON value is not appropriate for a given target type, or if a JSON number
+// overflows the target type, Unmarshal skips that field and completes the unmarshaling as best it can.
+// If no more serious errors are encountered, Unmarshal returns an UnmarshalTypeError describing the earliest such error.
+// In any case, it's not guaranteed that all the remaining fields following the problematic one will be unmarshaled into the target object.
+func Unmarshal(data []byte, v interface{}) error {
+	var err error
+	var dec *Decoder
+	switch vt := v.(type) {
+	case *string:
+		dec = borrowDecoder(nil, 0)
+		dec.length = len(data)
+		dec.data = data
+		err = dec.decodeString(vt)
+	case **string:
+		dec = borrowDecoder(nil, 0)
+		dec.length = len(data)
+		dec.data = data
+		err = dec.decodeStringNull(vt)
+	case *int:
+		dec = borrowDecoder(nil, 0)
+		dec.length = len(data)
+		dec.data = data
+		err = dec.decodeInt(vt)
+	case **int:
+		dec = borrowDecoder(nil, 0)
+		dec.length = len(data)
+		dec.data = data
+		err = dec.decodeIntNull(vt)
+	case *int8:
+		dec = borrowDecoder(nil, 0)
+		dec.length = len(data)
+		dec.data = data
+		err = dec.decodeInt8(vt)
+	case **int8:
+		dec = borrowDecoder(nil, 0)
+		dec.length = len(data)
+		dec.data = data
+		err = dec.decodeInt8Null(vt)
+	case *int16:
+		dec = borrowDecoder(nil, 0)
+		dec.length = len(data)
+		dec.data = data
+		err = dec.decodeInt16(vt)
+	case **int16:
+		dec = borrowDecoder(nil, 0)
+		dec.length = len(data)
+		dec.data = data
+		err = dec.decodeInt16Null(vt)
+	case *int32:
+		dec = borrowDecoder(nil, 0)
+		dec.length = len(data)
+		dec.data = data
+		err = dec.decodeInt32(vt)
+	case **int32:
+		dec = borrowDecoder(nil, 0)
+		dec.length = len(data)
+		dec.data = data
+		err = dec.decodeInt32Null(vt)
+	case *int64:
+		dec = borrowDecoder(nil, 0)
+		dec.length = len(data)
+		dec.data = data
+		err = dec.decodeInt64(vt)
+	case **int64:
+		dec = borrowDecoder(nil, 0)
+		dec.length = len(data)
+		dec.data = data
+		err = dec.decodeInt64Null(vt)
+	case *uint8:
+		dec = borrowDecoder(nil, 0)
+		dec.length = len(data)
+		dec.data = data
+		err = dec.decodeUint8(vt)
+	case **uint8:
+		dec = borrowDecoder(nil, 0)
+		dec.length = len(data)
+		dec.data = data
+		err = dec.decodeUint8Null(vt)
+	case *uint16:
+		dec = borrowDecoder(nil, 0)
+		dec.length = len(data)
+		dec.data = data
+		err = dec.decodeUint16(vt)
+	case **uint16:
+		dec = borrowDecoder(nil, 0)
+		dec.length = len(data)
+		dec.data = data
+		err = dec.decodeUint16Null(vt)
+	case *uint32:
+		dec = borrowDecoder(nil, 0)
+		dec.length = len(data)
+		dec.data = data
+		err = dec.decodeUint32(vt)
+	case **uint32:
+		dec = borrowDecoder(nil, 0)
+		dec.length = len(data)
+		dec.data = data
+		err = dec.decodeUint32Null(vt)
+	case *uint64:
+		dec = borrowDecoder(nil, 0)
+		dec.length = len(data)
+		dec.data = data
+		err = dec.decodeUint64(vt)
+	case **uint64:
+		dec = borrowDecoder(nil, 0)
+		dec.length = len(data)
+		dec.data = data
+		err = dec.decodeUint64Null(vt)
+	case *float64:
+		dec = borrowDecoder(nil, 0)
+		dec.length = len(data)
+		dec.data = data
+		err = dec.decodeFloat64(vt)
+	case **float64:
+		dec = borrowDecoder(nil, 0)
+		dec.length = len(data)
+		dec.data = data
+		err = dec.decodeFloat64Null(vt)
+	case *float32:
+		dec = borrowDecoder(nil, 0)
+		dec.length = len(data)
+		dec.data = data
+		err = dec.decodeFloat32(vt)
+	case **float32:
+		dec = borrowDecoder(nil, 0)
+		dec.length = len(data)
+		dec.data = data
+		err = dec.decodeFloat32Null(vt)
+	case *bool:
+		dec = borrowDecoder(nil, 0)
+		dec.length = len(data)
+		dec.data = data
+		err = dec.decodeBool(vt)
+	case **bool:
+		dec = borrowDecoder(nil, 0)
+		dec.length = len(data)
+		dec.data = data
+		err = dec.decodeBoolNull(vt)
+	case UnmarshalerJSONObject:
+		dec = borrowDecoder(nil, 0)
+		dec.length = len(data)
+		dec.data = make([]byte, len(data))
+		copy(dec.data, data)
+		_, err = dec.decodeObject(vt)
+	case UnmarshalerJSONArray:
+		dec = borrowDecoder(nil, 0)
+		dec.length = len(data)
+		dec.data = make([]byte, len(data))
+		copy(dec.data, data)
+		_, err = dec.decodeArray(vt)
+	case *interface{}:
+		dec = borrowDecoder(nil, 0)
+		dec.length = len(data)
+		dec.data = make([]byte, len(data))
+		copy(dec.data, data)
+		err = dec.decodeInterface(vt)
+	default:
+		return InvalidUnmarshalError(fmt.Sprintf(invalidUnmarshalErrorMsg, vt))
+	}
+	defer dec.Release()
+	if err != nil {
+		return err
+	}
+	return dec.err
+}
+
+// UnmarshalerJSONObject is the interface to implement to decode a JSON Object.
+type UnmarshalerJSONObject interface {
+	UnmarshalJSONObject(*Decoder, string) error
+	NKeys() int
+}
+
+// UnmarshalerJSONArray is the interface to implement to decode a JSON Array.
+type UnmarshalerJSONArray interface {
+	UnmarshalJSONArray(*Decoder) error
+}
+
+// A Decoder reads and decodes JSON values from an input stream.
+type Decoder struct {
+	r          io.Reader
+	data       []byte
+	err        error
+	isPooled   byte
+	called     byte
+	child      byte
+	cursor     int
+	length     int
+	keysDone   int
+	arrayIndex int
+}
+
+// Decode reads the next JSON-encoded value from the decoder's input (io.Reader) and stores it in the value pointed to by v.
+//
+// See the documentation for Unmarshal for details about the conversion of JSON into a Go value.
+// The differences between Decode and Unmarshal are:
+// 	- Decode reads from an io.Reader in the Decoder, whereas Unmarshal reads from a []byte
+// 	- Decode leaves to the user the option of borrowing and releasing a Decoder, whereas Unmarshal internally always borrows a Decoder and releases it when the unmarshaling is completed
+func (dec *Decoder) Decode(v interface{}) error {
+	if dec.isPooled == 1 {
+		panic(InvalidUsagePooledDecoderError("Invalid usage of pooled decoder"))
+	}
+	var err error
+	switch vt := v.(type) {
+	case *string:
+		err = dec.decodeString(vt)
+	case **string:
+		err = dec.decodeStringNull(vt)
+	case *int:
+		err = dec.decodeInt(vt)
+	case **int:
+		err = dec.decodeIntNull(vt)
+	case *int8:
+		err = dec.decodeInt8(vt)
+	case **int8:
+		err = dec.decodeInt8Null(vt)
+	case *int16:
+		err = dec.decodeInt16(vt)
+	case **int16:
+		err = dec.decodeInt16Null(vt)
+	case *int32:
+		err = dec.decodeInt32(vt)
+	case **int32:
+		err = dec.decodeInt32Null(vt)
+	case *int64:
+		err = dec.decodeInt64(vt)
+	case **int64:
+		err = dec.decodeInt64Null(vt)
+	case *uint8:
+		err = dec.decodeUint8(vt)
+	case **uint8:
+		err = dec.decodeUint8Null(vt)
+	case *uint16:
+		err = dec.decodeUint16(vt)
+	case **uint16:
+		err = dec.decodeUint16Null(vt)
+	case *uint32:
+		err = dec.decodeUint32(vt)
+	case **uint32:
+		err = dec.decodeUint32Null(vt)
+	case *uint64:
+		err = dec.decodeUint64(vt)
+	case **uint64:
+		err = dec.decodeUint64Null(vt)
+	case *float64:
+		err = dec.decodeFloat64(vt)
+	case **float64:
+		err = dec.decodeFloat64Null(vt)
+	case *float32:
+		err = dec.decodeFloat32(vt)
+	case **float32:
+		err = dec.decodeFloat32Null(vt)
+	case *bool:
+		err = dec.decodeBool(vt)
+	case **bool:
+		err = dec.decodeBoolNull(vt)
+	case UnmarshalerJSONObject:
+		_, err = dec.decodeObject(vt)
+	case UnmarshalerJSONArray:
+		_, err = dec.decodeArray(vt)
+	case *EmbeddedJSON:
+		err = dec.decodeEmbeddedJSON(vt)
+	case *interface{}:
+		err = dec.decodeInterface(vt)
+	default:
+		return InvalidUnmarshalError(fmt.Sprintf(invalidUnmarshalErrorMsg, vt))
+	}
+	if err != nil {
+		return err
+	}
+	return dec.err
+}
+
+// Non exported
+
+func isDigit(b byte) bool {
+	switch b {
+	case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
+		return true
+	default:
+		return false
+	}
+}
+
+func (dec *Decoder) read() bool {
+	if dec.r != nil {
+		// if we reach the end, double the buffer to ensure there's always more space
+		if len(dec.data) == dec.length {
+			nLen := dec.length * 2
+			if nLen == 0 {
+				nLen = 512
+			}
+			Buf := make([]byte, nLen, nLen)
+			copy(Buf, dec.data)
+			dec.data = Buf
+		}
+		var n int
+		var err error
+		for n == 0 {
+			n, err = dec.r.Read(dec.data[dec.length:])
+			if err != nil {
+				if err != io.EOF {
+					dec.err = err
+					return false
+				}
+				if n == 0 {
+					return false
+				}
+				dec.length = dec.length + n
+				return true
+			}
+		}
+		dec.length = dec.length + n
+		return true
+	}
+	return false
+}
+
+func (dec *Decoder) nextChar() byte {
+	for ; dec.cursor < dec.length || dec.read(); dec.cursor++ {
+		switch dec.data[dec.cursor] {
+		case ' ', '\n', '\t', '\r', ',':
+			continue
+		}
+		d := dec.data[dec.cursor]
+		return d
+	}
+	return 0
+}

vendor/github.com/francoispqt/gojay/decode_array.go

@@ -0,0 +1,247 @@
+package gojay
+
+import "reflect"
+
+// DecodeArray reads the next JSON-encoded value from the decoder's input (io.Reader)
+// and stores it in the value pointed to by v.
+//
+// v must implement UnmarshalerJSONArray.
+//
+// See the documentation for Unmarshal for details about the conversion of JSON into a Go value.
+func (dec *Decoder) DecodeArray(v UnmarshalerJSONArray) error {
+	if dec.isPooled == 1 {
+		panic(InvalidUsagePooledDecoderError("Invalid usage of pooled decoder"))
+	}
+	_, err := dec.decodeArray(v)
+	return err
+}
+func (dec *Decoder) decodeArray(arr UnmarshalerJSONArray) (int, error) {
+	// remember last array index in case of nested arrays
+	lastArrayIndex := dec.arrayIndex
+	dec.arrayIndex = 0
+	defer func() {
+		dec.arrayIndex = lastArrayIndex
+	}()
+	for ; dec.cursor < dec.length || dec.read(); dec.cursor++ {
+		switch dec.data[dec.cursor] {
+		case ' ', '\n', '\t', '\r', ',':
+			continue
+		case '[':
+			dec.cursor = dec.cursor + 1
+			// array is open, char is not space start readings
+			for dec.nextChar() != 0 {
+				// closing array
+				if dec.data[dec.cursor] == ']' {
+					dec.cursor = dec.cursor + 1
+					return dec.cursor, nil
+				}
+				// calling unmarshall function for each element of the slice
+				err := arr.UnmarshalJSONArray(dec)
+				if err != nil {
+					return 0, err
+				}
+				dec.arrayIndex++
+			}
+			return 0, dec.raiseInvalidJSONErr(dec.cursor)
+		case 'n':
+			// is null
+			dec.cursor++
+			err := dec.assertNull()
+			if err != nil {
+				return 0, err
+			}
+			return dec.cursor, nil
+		case '{', '"', 'f', 't', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
+			// can't unmarshall to struct
+			// we skip array and set Error
+			dec.err = dec.makeInvalidUnmarshalErr(arr)
+			err := dec.skipData()
+			if err != nil {
+				return 0, err
+			}
+			return dec.cursor, nil
+		default:
+			return 0, dec.raiseInvalidJSONErr(dec.cursor)
+		}
+	}
+	return 0, dec.raiseInvalidJSONErr(dec.cursor)
+}
+func (dec *Decoder) decodeArrayNull(v interface{}) (int, error) {
+	// remember last array index in case of nested arrays
+	lastArrayIndex := dec.arrayIndex
+	dec.arrayIndex = 0
+	defer func() {
+		dec.arrayIndex = lastArrayIndex
+	}()
+	vv := reflect.ValueOf(v)
+	vvt := vv.Type()
+	if vvt.Kind() != reflect.Ptr || vvt.Elem().Kind() != reflect.Ptr {
+		dec.err = ErrUnmarshalPtrExpected
+		return 0, dec.err
+	}
+	// not an array not an error, but do not know what to do
+	// do not check syntax
+	for ; dec.cursor < dec.length || dec.read(); dec.cursor++ {
+		switch dec.data[dec.cursor] {
+		case ' ', '\n', '\t', '\r', ',':
+			continue
+		case '[':
+			dec.cursor = dec.cursor + 1
+			// create our new type
+			elt := vv.Elem()
+			n := reflect.New(elt.Type().Elem())
+			var arr UnmarshalerJSONArray
+			var ok bool
+			if arr, ok = n.Interface().(UnmarshalerJSONArray); !ok {
+				dec.err = dec.makeInvalidUnmarshalErr((UnmarshalerJSONArray)(nil))
+				return 0, dec.err
+			}
+			// array is open, char is not space start readings
+			for dec.nextChar() != 0 {
+				// closing array
+				if dec.data[dec.cursor] == ']' {
+					elt.Set(n)
+					dec.cursor = dec.cursor + 1
+					return dec.cursor, nil
+				}
+				// calling unmarshall function for each element of the slice
+				err := arr.UnmarshalJSONArray(dec)
+				if err != nil {
+					return 0, err
+				}
+				dec.arrayIndex++
+			}
+			return 0, dec.raiseInvalidJSONErr(dec.cursor)
+		case 'n':
+			// is null
+			dec.cursor++
+			err := dec.assertNull()
+			if err != nil {
+				return 0, err
+			}
+			return dec.cursor, nil
+		case '{', '"', 'f', 't', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
+			// can't unmarshall to struct
+			// we skip array and set Error
+			dec.err = dec.makeInvalidUnmarshalErr((UnmarshalerJSONArray)(nil))
+			err := dec.skipData()
+			if err != nil {
+				return 0, err
+			}
+			return dec.cursor, nil
+		default:
+			return 0, dec.raiseInvalidJSONErr(dec.cursor)
+		}
+	}
+	return 0, dec.raiseInvalidJSONErr(dec.cursor)
+}
+
+func (dec *Decoder) skipArray() (int, error) {
+	var arraysOpen = 1
+	var arraysClosed = 0
+	// var stringOpen byte = 0
+	for j := dec.cursor; j < dec.length || dec.read(); j++ {
+		switch dec.data[j] {
+		case ']':
+			arraysClosed++
+			// everything is closed return
+			if arraysOpen == arraysClosed {
+				// add char to object data
+				return j + 1, nil
+			}
+		case '[':
+			arraysOpen++
+		case '"':
+			j++
+			var isInEscapeSeq bool
+			var isFirstQuote = true
+			for ; j < dec.length || dec.read(); j++ {
+				if dec.data[j] != '"' {
+					continue
+				}
+				if dec.data[j-1] != '\\' || (!isInEscapeSeq && !isFirstQuote) {
+					break
+				} else {
+					isInEscapeSeq = false
+				}
+				if isFirstQuote {
+					isFirstQuote = false
+				}
+				// loop backward and count how many anti slash found
+				// to see if string is effectively escaped
+				ct := 0
+				for i := j - 1; i > 0; i-- {
+					if dec.data[i] != '\\' {
+						break
+					}
+					ct++
+				}
+				// is pair number of slashes, quote is not escaped
+				if ct&1 == 0 {
+					break
+				}
+				isInEscapeSeq = true
+			}
+		default:
+			continue
+		}
+	}
+	return 0, dec.raiseInvalidJSONErr(dec.cursor)
+}
+
+// DecodeArrayFunc is a func type implementing UnmarshalerJSONArray.
+// Use it to cast a `func(*Decoder) error` to Unmarshal an array on the fly.
+
+type DecodeArrayFunc func(*Decoder) error
+
+// UnmarshalJSONArray implements UnmarshalerJSONArray.
+func (f DecodeArrayFunc) UnmarshalJSONArray(dec *Decoder) error {
+	return f(dec)
+}
+
+// IsNil implements UnmarshalerJSONArray.
+func (f DecodeArrayFunc) IsNil() bool {
+	return f == nil
+}
+
+// Add Values functions
+
+// AddArray decodes the JSON value within an object or an array to a UnmarshalerJSONArray.
+func (dec *Decoder) AddArray(v UnmarshalerJSONArray) error {
+	return dec.Array(v)
+}
+
+// AddArrayNull decodes the JSON value within an object or an array to a UnmarshalerJSONArray.
+func (dec *Decoder) AddArrayNull(v interface{}) error {
+	return dec.ArrayNull(v)
+}
+
+// Array decodes the JSON value within an object or an array to a UnmarshalerJSONArray.
+func (dec *Decoder) Array(v UnmarshalerJSONArray) error {
+	newCursor, err := dec.decodeArray(v)
+	if err != nil {
+		return err
+	}
+	dec.cursor = newCursor
+	dec.called |= 1
+	return nil
+}
+
+// ArrayNull decodes the JSON value within an object or an array to a UnmarshalerJSONArray.
+// v should be a pointer to an UnmarshalerJSONArray,
+// if `null` value is encountered in JSON, it will leave the value v untouched,
+// else it will create a new instance of the UnmarshalerJSONArray behind v.
+func (dec *Decoder) ArrayNull(v interface{}) error {
+	newCursor, err := dec.decodeArrayNull(v)
+	if err != nil {
+		return err
+	}
+	dec.cursor = newCursor
+	dec.called |= 1
+	return nil
+}
+
+// Index returns the index of an array being decoded.
+func (dec *Decoder) Index() int {
+	return dec.arrayIndex
+}

vendor/github.com/francoispqt/gojay/decode_bool.go

@@ -0,0 +1,241 @@
+package gojay
+
+// DecodeBool reads the next JSON-encoded value from the decoder's input (io.Reader)
+// and stores it in the boolean pointed to by v.
+//
+// See the documentation for Unmarshal for details about the conversion of JSON into a Go value.
+func (dec *Decoder) DecodeBool(v *bool) error {
+	if dec.isPooled == 1 {
+		panic(InvalidUsagePooledDecoderError("Invalid usage of pooled decoder"))
+	}
+	return dec.decodeBool(v)
+}
+func (dec *Decoder) decodeBool(v *bool) error {
+	for ; dec.cursor < dec.length || dec.read(); dec.cursor++ {
+		switch dec.data[dec.cursor] {
+		case ' ', '\n', '\t', '\r', ',':
+			continue
+		case 't':
+			dec.cursor++
+			err := dec.assertTrue()
+			if err != nil {
+				return err
+			}
+			*v = true
+			return nil
+		case 'f':
+			dec.cursor++
+			err := dec.assertFalse()
+			if err != nil {
+				return err
+			}
+			*v = false
+			return nil
+		case 'n':
+			dec.cursor++
+			err := dec.assertNull()
+			if err != nil {
+				return err
+			}
+			*v = false
+			return nil
+		default:
+			dec.err = dec.makeInvalidUnmarshalErr(v)
+			err := dec.skipData()
+			if err != nil {
+				return err
+			}
+			return nil
+		}
+	}
+	return nil
+}
+func (dec *Decoder) decodeBoolNull(v **bool) error {
+	for ; dec.cursor < dec.length || dec.read(); dec.cursor++ {
+		switch dec.data[dec.cursor] {
+		case ' ', '\n', '\t', '\r', ',':
+			continue
+		case 't':
+			dec.cursor++
+			err := dec.assertTrue()
+			if err != nil {
+				return err
+			}
+			if *v == nil {
+				*v = new(bool)
+			}
+			**v = true
+			return nil
+		case 'f':
+			dec.cursor++
+			err := dec.assertFalse()
+			if err != nil {
+				return err
+			}
+			if *v == nil {
+				*v = new(bool)
+			}
+			**v = false
+			return nil
+		case 'n':
+			dec.cursor++
+			err := dec.assertNull()
+			if err != nil {
+				return err
+			}
+			return nil
+		default:
+			dec.err = dec.makeInvalidUnmarshalErr(v)
+			err := dec.skipData()
+			if err != nil {
+				return err
+			}
+			return nil
+		}
+	}
+	return nil
+}
+
+func (dec *Decoder) assertTrue() error {
+	i := 0
+	for ; dec.cursor < dec.length || dec.read(); dec.cursor++ {
+		switch i {
+		case 0:
+			if dec.data[dec.cursor] != 'r' {
+				return dec.raiseInvalidJSONErr(dec.cursor)
+			}
+		case 1:
+			if dec.data[dec.cursor] != 'u' {
+				return dec.raiseInvalidJSONErr(dec.cursor)
+			}
+		case 2:
+			if dec.data[dec.cursor] != 'e' {
+				return dec.raiseInvalidJSONErr(dec.cursor)
+			}
+		case 3:
+			switch dec.data[dec.cursor] {
+			case ' ', '\b', '\t', '\n', ',', ']', '}':
+				// dec.cursor--
+				return nil
+			default:
+				return dec.raiseInvalidJSONErr(dec.cursor)
+			}
+		}
+		i++
+	}
+	if i == 3 {
+		return nil
+	}
+	return dec.raiseInvalidJSONErr(dec.cursor)
+}
+
+func (dec *Decoder) assertNull() error {
+	i := 0
+	for ; dec.cursor < dec.length || dec.read(); dec.cursor++ {
+		switch i {
+		case 0:
+			if dec.data[dec.cursor] != 'u' {
+				return dec.raiseInvalidJSONErr(dec.cursor)
+			}
+		case 1:
+			if dec.data[dec.cursor] != 'l' {
+				return dec.raiseInvalidJSONErr(dec.cursor)
+			}
+		case 2:
+			if dec.data[dec.cursor] != 'l' {
+				return dec.raiseInvalidJSONErr(dec.cursor)
+			}
+		case 3:
+			switch dec.data[dec.cursor] {
+			case ' ', '\t', '\n', ',', ']', '}':
+				// dec.cursor--
+				return nil
+			default:
+				return dec.raiseInvalidJSONErr(dec.cursor)
+			}
+		}
+		i++
+	}
+	if i == 3 {
+		return nil
+	}
+	return dec.raiseInvalidJSONErr(dec.cursor)
+}
+
+func (dec *Decoder) assertFalse() error {
+	i := 0
+	for ; dec.cursor < dec.length || dec.read(); dec.cursor++ {
+		switch i {
+		case 0:
+			if dec.data[dec.cursor] != 'a' {
+				return dec.raiseInvalidJSONErr(dec.cursor)
+			}
+		case 1:
+			if dec.data[dec.cursor] != 'l' {
+				return dec.raiseInvalidJSONErr(dec.cursor)
+			}
+		case 2:
+			if dec.data[dec.cursor] != 's' {
+				return dec.raiseInvalidJSONErr(dec.cursor)
+			}
+		case 3:
+			if dec.data[dec.cursor] != 'e' {
+				return dec.raiseInvalidJSONErr(dec.cursor)
+			}
+		case 4:
+			switch dec.data[dec.cursor] {
+			case ' ', '\t', '\n', ',', ']', '}':
+				// dec.cursor--
+				return nil
+			default:
+				return dec.raiseInvalidJSONErr(dec.cursor)
+			}
+		}
+		i++
+	}
+	if i == 4 {
+		return nil
+	}
+	return dec.raiseInvalidJSONErr(dec.cursor)
+}
+
+// Add Values functions
+
+// AddBool decodes the JSON value within an object or an array to a *bool.
+// If next key is neither null nor a JSON boolean, an InvalidUnmarshalError will be returned.
+// If next key is null, bool will be false.
+func (dec *Decoder) AddBool(v *bool) error {
+	return dec.Bool(v)
+}
+
+// AddBoolNull decodes the JSON value within an object or an array to a *bool.
+// If next key is neither null nor a JSON boolean, an InvalidUnmarshalError will be returned.
+// If next key is null, bool will be false.
+// If a `null` is encountered, gojay does not change the value of the pointer.
+func (dec *Decoder) AddBoolNull(v **bool) error {
+	return dec.BoolNull(v)
+}
+
+// Bool decodes the JSON value within an object or an array to a *bool.
+// If next key is neither null nor a JSON boolean, an InvalidUnmarshalError will be returned.
+// If next key is null, bool will be false.
+func (dec *Decoder) Bool(v *bool) error {
+	err := dec.decodeBool(v)
+	if err != nil {
+		return err
+	}
+	dec.called |= 1
+	return nil
+}
+
+// BoolNull decodes the JSON value within an object or an array to a *bool.
+// If next key is neither null nor a JSON boolean, an InvalidUnmarshalError will be returned.
+// If next key is null, bool will be false.
+func (dec *Decoder) BoolNull(v **bool) error {
+	err := dec.decodeBoolNull(v)
+	if err != nil {
+		return err
+	}
+	dec.called |= 1
+	return nil
+}

vendor/github.com/francoispqt/gojay/decode_embedded_json.go

@@ -0,0 +1,85 @@
+package gojay
+
+// EmbeddedJSON is a raw encoded JSON value.
+// It can be used to delay JSON decoding or precompute a JSON encoding.
+type EmbeddedJSON []byte
+
+func (dec *Decoder) decodeEmbeddedJSON(ej *EmbeddedJSON) error {
+	var err error
+	if ej == nil {
+		return InvalidUnmarshalError("Invalid nil pointer given")
+	}
+	var beginOfEmbeddedJSON int
+	for ; dec.cursor < dec.length || dec.read(); dec.cursor++ {
+		switch dec.data[dec.cursor] {
+		case ' ', '\n', '\t', '\r', ',':
+			continue
+		// is null
+		case 'n':
+			beginOfEmbeddedJSON = dec.cursor
+			dec.cursor++
+			err := dec.assertNull()
+			if err != nil {
+				return err
+			}
+		case 't':
+			beginOfEmbeddedJSON = dec.cursor
+			dec.cursor++
+			err := dec.assertTrue()
+			if err != nil {
+				return err
+			}
+		// is false
+		case 'f':
+			beginOfEmbeddedJSON = dec.cursor
+			dec.cursor++
+			err := dec.assertFalse()
+			if err != nil {
+				return err
+			}
+		// is an object
+		case '{':
+			beginOfEmbeddedJSON = dec.cursor
+			dec.cursor = dec.cursor + 1
+			dec.cursor, err = dec.skipObject()
+		// is string
+		case '"':
+			beginOfEmbeddedJSON = dec.cursor
+			dec.cursor = dec.cursor + 1
+			err = dec.skipString() // why no new dec.cursor in result?
+		// is array
+		case '[':
+			beginOfEmbeddedJSON = dec.cursor
+			dec.cursor = dec.cursor + 1
+			dec.cursor, err = dec.skipArray()
+		case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-':
+			beginOfEmbeddedJSON = dec.cursor
+			dec.cursor, err = dec.skipNumber()
+		}
+		break
+	}
+	if err == nil {
+		if dec.cursor-1 >= beginOfEmbeddedJSON {
+			*ej = append(*ej, dec.data[beginOfEmbeddedJSON:dec.cursor]...)
+		}
+		dec.called |= 1
+	}
+	return err
+}
+
+// AddEmbeddedJSON adds an EmbeddedsJSON to the value pointed by v.
+// It can be used to delay JSON decoding or precompute a JSON encoding.
+func (dec *Decoder) AddEmbeddedJSON(v *EmbeddedJSON) error {
+	return dec.EmbeddedJSON(v)
+}
+
+// EmbeddedJSON adds an EmbeddedsJSON to the value pointed by v.
+// It can be used to delay JSON decoding or precompute a JSON encoding.
+func (dec *Decoder) EmbeddedJSON(v *EmbeddedJSON) error {
+	err := dec.decodeEmbeddedJSON(v)
+	if err != nil {
+		return err
+	}
+	dec.called |= 1
+	return nil
+}

vendor/github.com/francoispqt/gojay/decode_interface.go

@@ -0,0 +1,130 @@
+package gojay
+
+// TODO @afiune for now we are using the standard json unmarshaling but in
+// the future it would be great to implement one here inside this repo
+import "encoding/json"
+
+// DecodeInterface reads the next JSON-encoded value from the decoder's input (io.Reader) and stores it in the value pointed to by i.
+//
+// i must be an interface poiter
+func (dec *Decoder) DecodeInterface(i *interface{}) error {
+	if dec.isPooled == 1 {
+		panic(InvalidUsagePooledDecoderError("Invalid usage of pooled decoder"))
+	}
+	err := dec.decodeInterface(i)
+	return err
+}
+
+func (dec *Decoder) decodeInterface(i *interface{}) error {
+	start, end, err := dec.getObject()
+	if err != nil {
+		dec.cursor = start
+		return err
+	}
+
+	// if start & end are equal the object is a null, don't unmarshal
+	if start == end {
+		return nil
+	}
+
+	object := dec.data[start:end]
+	if err = json.Unmarshal(object, i); err != nil {
+		return err
+	}
+
+	dec.cursor = end
+	return nil
+}
+
+// @afiune Maybe return the type as well?
+func (dec *Decoder) getObject() (start int, end int, err error) {
+	// start cursor
+	for ; dec.cursor < dec.length || dec.read(); dec.cursor++ {
+		switch dec.data[dec.cursor] {
+		case ' ', '\n', '\t', '\r', ',':
+			continue
+		// is null
+		case 'n':
+			dec.cursor++
+			err = dec.assertNull()
+			if err != nil {
+				return
+			}
+			// Set start & end to the same cursor to indicate the object
+			// is a null and should not be unmarshal
+			start = dec.cursor
+			end = dec.cursor
+			return
+		case 't':
+			start = dec.cursor
+			dec.cursor++
+			err = dec.assertTrue()
+			if err != nil {
+				return
+			}
+			end = dec.cursor
+			dec.cursor++
+			return
+		// is false
+		case 'f':
+			start = dec.cursor
+			dec.cursor++
+			err = dec.assertFalse()
+			if err != nil {
+				return
+			}
+			end = dec.cursor
+			dec.cursor++
+			return
+		// is an object
+		case '{':
+			start = dec.cursor
+			dec.cursor++
+			end, err = dec.skipObject()
+			dec.cursor = end
+			return
+		// is string
+		case '"':
+			start = dec.cursor
+			dec.cursor++
+			start, end, err = dec.getString()
+			start--
+			dec.cursor = end
+			return
+		// is array
+		case '[':
+			start = dec.cursor
+			dec.cursor++
+			end, err = dec.skipArray()
+			dec.cursor = end
+			return
+		case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-':
+			start = dec.cursor
+			end, err = dec.skipNumber()
+			dec.cursor = end
+			return
+		default:
+			err = dec.raiseInvalidJSONErr(dec.cursor)
+			return
+		}
+	}
+	err = dec.raiseInvalidJSONErr(dec.cursor)
+	return
+}
+
+// Add Values functions
+
+// AddInterface decodes the JSON value within an object or an array to a interface{}.
+func (dec *Decoder) AddInterface(v *interface{}) error {
+	return dec.Interface(v)
+}
+
+// Interface decodes the JSON value within an object or an array to an interface{}.
+func (dec *Decoder) Interface(value *interface{}) error {
+	err := dec.decodeInterface(value)
+	if err != nil {
+		return err
+	}
+	dec.called |= 1
+	return nil
+}