2017-11-04 06:55:25 -07:00
|
|
|
package helper
|
|
|
|
|
|
|
|
import (
|
|
|
|
"bytes"
|
2018-02-02 16:11:11 -08:00
|
|
|
"fmt"
|
2019-02-26 15:41:50 -08:00
|
|
|
"image/png"
|
2017-11-04 06:55:25 -07:00
|
|
|
"io"
|
Support Telegram animated stickers (tgs) format (#1173)
This is half a fix for #874
This patch introduces a new config flag:
- MediaConvertTgs
These need to be treated independently from the existing
MediaConvertWebPToPNG flag because Tgs→WebP results in an
*animated* WebP, and the WebP→PNG converter can't handle
animated WebP files yet.
Furthermore, some platforms (like discord) don't even support
animated WebP files, so the user may want to fall back to
static PNGs (not APNGs).
The final reason why this is only half a fix is that this
introduces an external dependency, namely lottie, to be
installed like this:
$ pip3 install lottie cairosvg
This patch works by writing the tgs to a temporary file in /tmp,
calling lottie to convert it (this conversion may take several seconds!),
and then deleting the temporary file.
The temporary file is absolutely necessary, as lottie refuses to
work on non-seekable files.
If anyone comes up with a reasonable use case where /tmp is
unavailable, I can add yet another config option for that, if desired.
Telegram will bail out if the option is configured but lottie isn't found.
2020-08-23 13:34:28 -07:00
|
|
|
"io/ioutil"
|
2017-11-04 06:55:25 -07:00
|
|
|
"net/http"
|
Support Telegram animated stickers (tgs) format (#1173)
This is half a fix for #874
This patch introduces a new config flag:
- MediaConvertTgs
These need to be treated independently from the existing
MediaConvertWebPToPNG flag because Tgs→WebP results in an
*animated* WebP, and the WebP→PNG converter can't handle
animated WebP files yet.
Furthermore, some platforms (like discord) don't even support
animated WebP files, so the user may want to fall back to
static PNGs (not APNGs).
The final reason why this is only half a fix is that this
introduces an external dependency, namely lottie, to be
installed like this:
$ pip3 install lottie cairosvg
This patch works by writing the tgs to a temporary file in /tmp,
calling lottie to convert it (this conversion may take several seconds!),
and then deleting the temporary file.
The temporary file is absolutely necessary, as lottie refuses to
work on non-seekable files.
If anyone comes up with a reasonable use case where /tmp is
unavailable, I can add yet another config option for that, if desired.
Telegram will bail out if the option is configured but lottie isn't found.
2020-08-23 13:34:28 -07:00
|
|
|
"os"
|
|
|
|
"os/exec"
|
2018-06-09 05:35:02 -07:00
|
|
|
"regexp"
|
2018-04-19 04:04:12 -07:00
|
|
|
"strings"
|
2017-11-04 06:55:25 -07:00
|
|
|
"time"
|
2018-07-21 15:27:49 -07:00
|
|
|
"unicode/utf8"
|
2018-06-09 03:47:40 -07:00
|
|
|
|
2019-02-26 15:41:50 -08:00
|
|
|
"golang.org/x/image/webp"
|
|
|
|
|
2018-06-09 03:47:40 -07:00
|
|
|
"github.com/42wim/matterbridge/bridge/config"
|
2019-11-17 12:18:01 -08:00
|
|
|
"github.com/gomarkdown/markdown"
|
2020-02-02 09:35:43 -08:00
|
|
|
"github.com/gomarkdown/markdown/html"
|
2019-11-17 12:18:01 -08:00
|
|
|
"github.com/gomarkdown/markdown/parser"
|
2018-12-26 06:16:09 -08:00
|
|
|
"github.com/sirupsen/logrus"
|
2017-11-04 06:55:25 -07:00
|
|
|
)
|
|
|
|
|
2019-02-23 13:51:27 -08:00
|
|
|
// DownloadFile downloads the given non-authenticated URL.
|
2017-11-04 06:55:25 -07:00
|
|
|
func DownloadFile(url string) (*[]byte, error) {
|
2018-02-24 14:35:53 -08:00
|
|
|
return DownloadFileAuth(url, "")
|
|
|
|
}
|
|
|
|
|
2019-02-23 13:51:27 -08:00
|
|
|
// DownloadFileAuth downloads the given URL using the specified authentication token.
|
2018-02-24 14:35:53 -08:00
|
|
|
func DownloadFileAuth(url string, auth string) (*[]byte, error) {
|
2017-11-04 06:55:25 -07:00
|
|
|
var buf bytes.Buffer
|
|
|
|
client := &http.Client{
|
|
|
|
Timeout: time.Second * 5,
|
|
|
|
}
|
|
|
|
req, err := http.NewRequest("GET", url, nil)
|
2018-02-24 14:35:53 -08:00
|
|
|
if auth != "" {
|
|
|
|
req.Header.Add("Authorization", auth)
|
|
|
|
}
|
2017-11-04 06:55:25 -07:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
resp, err := client.Do(req)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2018-02-06 15:04:02 -08:00
|
|
|
defer resp.Body.Close()
|
2017-11-04 06:55:25 -07:00
|
|
|
io.Copy(&buf, resp.Body)
|
|
|
|
data := buf.Bytes()
|
|
|
|
return &data, nil
|
|
|
|
}
|
2017-11-24 14:27:13 -08:00
|
|
|
|
2018-11-14 13:43:52 -08:00
|
|
|
// GetSubLines splits messages in newline-delimited lines. If maxLineLength is
|
2019-02-23 13:51:27 -08:00
|
|
|
// specified as non-zero GetSubLines will also clip long lines to the maximum
|
|
|
|
// length and insert a warning marker that the line was clipped.
|
2018-11-14 13:43:52 -08:00
|
|
|
//
|
|
|
|
// TODO: The current implementation has the inconvenient that it disregards
|
|
|
|
// word boundaries when splitting but this is hard to solve without potentially
|
|
|
|
// breaking formatting and other stylistic effects.
|
|
|
|
func GetSubLines(message string, maxLineLength int) []string {
|
|
|
|
const clippingMessage = " <clipped message>"
|
|
|
|
|
|
|
|
var lines []string
|
|
|
|
for _, line := range strings.Split(strings.TrimSpace(message), "\n") {
|
|
|
|
if maxLineLength == 0 || len([]byte(line)) <= maxLineLength {
|
|
|
|
lines = append(lines, line)
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
// !!! WARNING !!!
|
|
|
|
// Before touching the splitting logic below please ensure that you PROPERLY
|
|
|
|
// understand how strings, runes and range loops over strings work in Go.
|
|
|
|
// A good place to start is to read https://blog.golang.org/strings. :-)
|
|
|
|
var splitStart int
|
|
|
|
var startOfPreviousRune int
|
|
|
|
for i := range line {
|
|
|
|
if i-splitStart > maxLineLength-len([]byte(clippingMessage)) {
|
|
|
|
lines = append(lines, line[splitStart:startOfPreviousRune]+clippingMessage)
|
|
|
|
splitStart = startOfPreviousRune
|
|
|
|
}
|
|
|
|
startOfPreviousRune = i
|
2017-11-24 14:27:13 -08:00
|
|
|
}
|
2018-11-14 13:43:52 -08:00
|
|
|
// This last append is safe to do without looking at the remaining byte-length
|
|
|
|
// as we assume that the byte-length of the last rune will never exceed that of
|
|
|
|
// the byte-length of the clipping message.
|
|
|
|
lines = append(lines, line[splitStart:])
|
2017-11-24 14:27:13 -08:00
|
|
|
}
|
2018-11-14 13:43:52 -08:00
|
|
|
return lines
|
2017-11-24 14:27:13 -08:00
|
|
|
}
|
2018-02-02 16:11:11 -08:00
|
|
|
|
2019-02-23 13:51:27 -08:00
|
|
|
// HandleExtra manages the supplementary details stored inside a message's 'Extra' field map.
|
2018-02-02 16:11:11 -08:00
|
|
|
func HandleExtra(msg *config.Message, general *config.Protocol) []config.Message {
|
|
|
|
extra := msg.Extra
|
|
|
|
rmsg := []config.Message{}
|
2018-11-15 11:43:43 -08:00
|
|
|
for _, f := range extra[config.EventFileFailureSize] {
|
2018-11-07 12:35:59 -08:00
|
|
|
fi := f.(config.FileInfo)
|
|
|
|
text := fmt.Sprintf("file %s too big to download (%#v > allowed size: %#v)", fi.Name, fi.Size, general.MediaDownloadSize)
|
2019-02-23 13:51:27 -08:00
|
|
|
rmsg = append(rmsg, config.Message{
|
|
|
|
Text: text,
|
|
|
|
Username: "<system> ",
|
|
|
|
Channel: msg.Channel,
|
|
|
|
Account: msg.Account,
|
|
|
|
})
|
2018-02-02 16:11:11 -08:00
|
|
|
}
|
|
|
|
return rmsg
|
|
|
|
}
|
2018-02-19 15:54:35 -08:00
|
|
|
|
2019-02-23 13:51:27 -08:00
|
|
|
// GetAvatar constructs a URL for a given user-avatar if it is available in the cache.
|
2018-02-19 15:54:35 -08:00
|
|
|
func GetAvatar(av map[string]string, userid string, general *config.Protocol) string {
|
|
|
|
if sha, ok := av[userid]; ok {
|
2018-02-20 08:15:54 -08:00
|
|
|
return general.MediaServerDownload + "/" + sha + "/" + userid + ".png"
|
2018-02-19 15:54:35 -08:00
|
|
|
}
|
|
|
|
return ""
|
|
|
|
}
|
2018-02-23 14:07:23 -08:00
|
|
|
|
2019-02-23 13:51:27 -08:00
|
|
|
// HandleDownloadSize checks a specified filename against the configured download blacklist
|
|
|
|
// and checks a specified file-size against the configure limit.
|
|
|
|
func HandleDownloadSize(logger *logrus.Entry, msg *config.Message, name string, size int64, general *config.Protocol) error {
|
2018-06-09 05:35:02 -07:00
|
|
|
// check blacklist here
|
|
|
|
for _, entry := range general.MediaDownloadBlackList {
|
|
|
|
if entry != "" {
|
|
|
|
re, err := regexp.Compile(entry)
|
|
|
|
if err != nil {
|
2019-02-23 13:51:27 -08:00
|
|
|
logger.Errorf("incorrect regexp %s for %s", entry, msg.Account)
|
2018-06-09 05:35:02 -07:00
|
|
|
continue
|
|
|
|
}
|
|
|
|
if re.MatchString(name) {
|
|
|
|
return fmt.Errorf("Matching blacklist %s. Not downloading %s", entry, name)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2019-02-23 13:51:27 -08:00
|
|
|
logger.Debugf("Trying to download %#v with size %#v", name, size)
|
2018-02-23 14:07:23 -08:00
|
|
|
if int(size) > general.MediaDownloadSize {
|
2018-11-15 11:43:43 -08:00
|
|
|
msg.Event = config.EventFileFailureSize
|
2019-02-23 13:51:27 -08:00
|
|
|
msg.Extra[msg.Event] = append(msg.Extra[msg.Event], config.FileInfo{
|
|
|
|
Name: name,
|
|
|
|
Comment: msg.Text,
|
|
|
|
Size: size,
|
|
|
|
})
|
2018-02-23 14:07:23 -08:00
|
|
|
return fmt.Errorf("File %#v to large to download (%#v). MediaDownloadSize is %#v", name, size, general.MediaDownloadSize)
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2019-02-23 13:51:27 -08:00
|
|
|
// HandleDownloadData adds the data for a remote file into a Matterbridge gateway message.
|
|
|
|
func HandleDownloadData(logger *logrus.Entry, msg *config.Message, name, comment, url string, data *[]byte, general *config.Protocol) {
|
2018-02-23 14:07:23 -08:00
|
|
|
var avatar bool
|
2019-02-23 13:51:27 -08:00
|
|
|
logger.Debugf("Download OK %#v %#v", name, len(*data))
|
2018-11-15 11:43:43 -08:00
|
|
|
if msg.Event == config.EventAvatarDownload {
|
2018-02-23 14:07:23 -08:00
|
|
|
avatar = true
|
|
|
|
}
|
2019-02-23 13:51:27 -08:00
|
|
|
msg.Extra["file"] = append(msg.Extra["file"], config.FileInfo{
|
|
|
|
Name: name,
|
|
|
|
Data: data,
|
|
|
|
URL: url,
|
|
|
|
Comment: comment,
|
|
|
|
Avatar: avatar,
|
|
|
|
})
|
2018-02-23 14:07:23 -08:00
|
|
|
}
|
2018-04-19 03:53:49 -07:00
|
|
|
|
2019-02-23 13:51:27 -08:00
|
|
|
var emptyLineMatcher = regexp.MustCompile("\n+")
|
|
|
|
|
|
|
|
// RemoveEmptyNewLines collapses consecutive newline characters into a single one and
|
|
|
|
// trims any preceding or trailing newline characters as well.
|
2018-04-19 03:53:49 -07:00
|
|
|
func RemoveEmptyNewLines(msg string) string {
|
2019-02-23 13:51:27 -08:00
|
|
|
return emptyLineMatcher.ReplaceAllString(strings.Trim(msg, "\n"), "\n")
|
2018-04-19 03:53:49 -07:00
|
|
|
}
|
2018-07-21 15:27:49 -07:00
|
|
|
|
2019-02-23 13:51:27 -08:00
|
|
|
// ClipMessage trims a message to the specified length if it exceeds it and adds a warning
|
|
|
|
// to the message in case it does so.
|
2018-07-21 15:27:49 -07:00
|
|
|
func ClipMessage(text string, length int) string {
|
2019-02-23 13:51:27 -08:00
|
|
|
const clippingMessage = " <clipped message>"
|
2018-07-21 15:27:49 -07:00
|
|
|
if len(text) > length {
|
2019-02-23 13:51:27 -08:00
|
|
|
text = text[:length-len(clippingMessage)]
|
2018-07-21 15:27:49 -07:00
|
|
|
if r, size := utf8.DecodeLastRuneInString(text); r == utf8.RuneError {
|
|
|
|
text = text[:len(text)-size]
|
|
|
|
}
|
2019-02-23 13:51:27 -08:00
|
|
|
text += clippingMessage
|
2018-07-21 15:27:49 -07:00
|
|
|
}
|
|
|
|
return text
|
|
|
|
}
|
2019-01-06 13:25:19 -08:00
|
|
|
|
2019-11-17 12:18:01 -08:00
|
|
|
// ParseMarkdown takes in an input string as markdown and parses it to html
|
2019-01-06 13:25:19 -08:00
|
|
|
func ParseMarkdown(input string) string {
|
2020-12-06 10:38:32 -08:00
|
|
|
extensions := parser.HardLineBreak | parser.NoIntraEmphasis | parser.FencedCode
|
2019-11-17 12:18:01 -08:00
|
|
|
markdownParser := parser.NewWithExtensions(extensions)
|
2020-02-02 09:35:43 -08:00
|
|
|
renderer := html.NewRenderer(html.RendererOptions{
|
|
|
|
Flags: 0,
|
|
|
|
})
|
|
|
|
parsedMarkdown := markdown.ToHTML([]byte(input), markdownParser, renderer)
|
2019-11-17 12:18:01 -08:00
|
|
|
res := string(parsedMarkdown)
|
2019-03-02 15:29:29 -08:00
|
|
|
res = strings.TrimPrefix(res, "<p>")
|
|
|
|
res = strings.TrimSuffix(res, "</p>\n")
|
|
|
|
return res
|
2019-01-06 13:25:19 -08:00
|
|
|
}
|
2019-02-26 15:41:50 -08:00
|
|
|
|
Support Telegram animated stickers (tgs) format (#1173)
This is half a fix for #874
This patch introduces a new config flag:
- MediaConvertTgs
These need to be treated independently from the existing
MediaConvertWebPToPNG flag because Tgs→WebP results in an
*animated* WebP, and the WebP→PNG converter can't handle
animated WebP files yet.
Furthermore, some platforms (like discord) don't even support
animated WebP files, so the user may want to fall back to
static PNGs (not APNGs).
The final reason why this is only half a fix is that this
introduces an external dependency, namely lottie, to be
installed like this:
$ pip3 install lottie cairosvg
This patch works by writing the tgs to a temporary file in /tmp,
calling lottie to convert it (this conversion may take several seconds!),
and then deleting the temporary file.
The temporary file is absolutely necessary, as lottie refuses to
work on non-seekable files.
If anyone comes up with a reasonable use case where /tmp is
unavailable, I can add yet another config option for that, if desired.
Telegram will bail out if the option is configured but lottie isn't found.
2020-08-23 13:34:28 -07:00
|
|
|
// ConvertWebPToPNG converts input data (which should be WebP format) to PNG format
|
2019-02-26 15:41:50 -08:00
|
|
|
func ConvertWebPToPNG(data *[]byte) error {
|
|
|
|
r := bytes.NewReader(*data)
|
|
|
|
m, err := webp.Decode(r)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
var output []byte
|
|
|
|
w := bytes.NewBuffer(output)
|
|
|
|
if err := png.Encode(w, m); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
*data = w.Bytes()
|
|
|
|
return nil
|
|
|
|
}
|
Support Telegram animated stickers (tgs) format (#1173)
This is half a fix for #874
This patch introduces a new config flag:
- MediaConvertTgs
These need to be treated independently from the existing
MediaConvertWebPToPNG flag because Tgs→WebP results in an
*animated* WebP, and the WebP→PNG converter can't handle
animated WebP files yet.
Furthermore, some platforms (like discord) don't even support
animated WebP files, so the user may want to fall back to
static PNGs (not APNGs).
The final reason why this is only half a fix is that this
introduces an external dependency, namely lottie, to be
installed like this:
$ pip3 install lottie cairosvg
This patch works by writing the tgs to a temporary file in /tmp,
calling lottie to convert it (this conversion may take several seconds!),
and then deleting the temporary file.
The temporary file is absolutely necessary, as lottie refuses to
work on non-seekable files.
If anyone comes up with a reasonable use case where /tmp is
unavailable, I can add yet another config option for that, if desired.
Telegram will bail out if the option is configured but lottie isn't found.
2020-08-23 13:34:28 -07:00
|
|
|
|
|
|
|
// CanConvertTgsToX Checks whether the external command necessary for ConvertTgsToX works.
|
|
|
|
func CanConvertTgsToX() error {
|
|
|
|
// We depend on the fact that `lottie_convert.py --help` has exit status 0.
|
|
|
|
// Hyrum's Law predicted this, and Murphy's Law predicts that this will break eventually.
|
|
|
|
// However, there is no alternative like `lottie_convert.py --is-properly-installed`
|
|
|
|
cmd := exec.Command("lottie_convert.py", "--help")
|
|
|
|
return cmd.Run()
|
|
|
|
}
|
|
|
|
|
|
|
|
// ConvertTgsToWebP convert input data (which should be tgs format) to WebP format
|
|
|
|
// This relies on an external command, which is ugly, but works.
|
|
|
|
func ConvertTgsToX(data *[]byte, outputFormat string, logger *logrus.Entry) error {
|
|
|
|
// lottie can't handle input from a pipe, so write to a temporary file:
|
|
|
|
tmpFile, err := ioutil.TempFile(os.TempDir(), "matterbridge-lottie-*.tgs")
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
tmpFileName := tmpFile.Name()
|
|
|
|
defer func() {
|
|
|
|
if removeErr := os.Remove(tmpFileName); removeErr != nil {
|
|
|
|
logger.Errorf("Could not delete temporary file %s: %v", tmpFileName, removeErr)
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
|
|
|
|
if _, writeErr := tmpFile.Write(*data); writeErr != nil {
|
|
|
|
return writeErr
|
|
|
|
}
|
|
|
|
// Must close before calling lottie to avoid data races:
|
|
|
|
if closeErr := tmpFile.Close(); closeErr != nil {
|
|
|
|
return closeErr
|
|
|
|
}
|
|
|
|
|
|
|
|
// Call lottie to transform:
|
|
|
|
cmd := exec.Command("lottie_convert.py", "--input-format", "lottie", "--output-format", outputFormat, tmpFileName, "/dev/stdout")
|
|
|
|
cmd.Stderr = nil
|
|
|
|
// NB: lottie writes progress into to stderr in all cases.
|
|
|
|
stdout, stderr := cmd.Output()
|
|
|
|
if stderr != nil {
|
|
|
|
// 'stderr' already contains some parts of Stderr, because it was set to 'nil'.
|
|
|
|
return stderr
|
|
|
|
}
|
|
|
|
|
|
|
|
*data = stdout
|
|
|
|
return nil
|
|
|
|
}
|