uwaru/matterbridge
1
0
Fork 0
forked from jshiffer/matterbridge
Code Pull Requests Activity

Update dependencies (#1800)

Browse Source
This commit is contained in:
Wim 2022-04-12 00:30:21 +02:00 committed by GitHub
parent f044b948e2
commit 281ef53e7d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
93 changed files with 5743 additions and 1749 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
go.mod
View File

@ -6,7 +6,7 @@ require (
github.com/Benau/tgsconverter v0.0.0-20210809170556-99f4a4f6337f github.com/Benau/tgsconverter v0.0.0-20210809170556-99f4a4f6337f
github.com/Philipp15b/go-steam v1.0.1-0.20200727090957-6ae9b3c0a560 github.com/Philipp15b/go-steam v1.0.1-0.20200727090957-6ae9b3c0a560
github.com/Rhymen/go-whatsapp v0.1.2-0.20211102134409-31a2e740845c github.com/Rhymen/go-whatsapp v0.1.2-0.20211102134409-31a2e740845c
github.com/SevereCloud/vksdk/v2 v2.13.1 github.com/SevereCloud/vksdk/v2 v2.14.0
github.com/bwmarrin/discordgo v0.24.0 github.com/bwmarrin/discordgo v0.24.0
github.com/d5/tengo/v2 v2.10.1 github.com/d5/tengo/v2 v2.10.1
github.com/davecgh/go-spew v1.1.1 github.com/davecgh/go-spew v1.1.1
@ -22,7 +22,7 @@ require (
github.com/keybase/go-keybase-chat-bot v0.0.0-20211201215354-ee4b23828b55 github.com/keybase/go-keybase-chat-bot v0.0.0-20211201215354-ee4b23828b55
github.com/kyokomi/emoji/v2 v2.2.9 github.com/kyokomi/emoji/v2 v2.2.9
github.com/labstack/echo/v4 v4.7.2 github.com/labstack/echo/v4 v4.7.2
github.com/lrstanley/girc v0.0.0-20220321215535-9664730c7858 github.com/lrstanley/girc v0.0.0-20220409202343-de3f963fb827
github.com/matterbridge/Rocket.Chat.Go.SDK v0.0.0-20211016222428-79310a412696 github.com/matterbridge/Rocket.Chat.Go.SDK v0.0.0-20211016222428-79310a412696
github.com/matterbridge/go-xmpp v0.0.0-20211030125215-791a06c5f1be github.com/matterbridge/go-xmpp v0.0.0-20211030125215-791a06c5f1be
github.com/matterbridge/gomatrix v0.0.0-20220205235239-607eb9ee6419 github.com/matterbridge/gomatrix v0.0.0-20220205235239-607eb9ee6419
@ -48,14 +48,14 @@ require (
github.com/yaegashi/msgraph.go v0.1.4 github.com/yaegashi/msgraph.go v0.1.4
github.com/zfjagann/golang-ring v0.0.0-20210116075443-7c86fdb43134 github.com/zfjagann/golang-ring v0.0.0-20210116075443-7c86fdb43134
go.mau.fi/whatsmeow v0.0.0-20220329131721-9f73bc00d158 go.mau.fi/whatsmeow v0.0.0-20220329131721-9f73bc00d158
golang.org/x/image v0.0.0-20220302094943-723b81ca9867 golang.org/x/image v0.0.0-20220321031419-a8550c1d254a
golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a
golang.org/x/text v0.3.7 golang.org/x/text v0.3.7
gomod.garykim.dev/nc-talk v0.3.0 gomod.garykim.dev/nc-talk v0.3.0
google.golang.org/protobuf v1.27.1 google.golang.org/protobuf v1.28.0
gopkg.in/olahol/melody.v1 v1.0.0-20170518105555-d52139073376 gopkg.in/olahol/melody.v1 v1.0.0-20170518105555-d52139073376
layeh.com/gumble v0.0.0-20200818122324-146f9205029b layeh.com/gumble v0.0.0-20200818122324-146f9205029b
modernc.org/sqlite v1.15.4 modernc.org/sqlite v1.16.0
) )
require ( require (
@ -80,7 +80,7 @@ require (
github.com/json-iterator/go v1.1.12 // indirect github.com/json-iterator/go v1.1.12 // indirect
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
github.com/kettek/apng v0.0.0-20191108220231-414630eed80f // indirect github.com/kettek/apng v0.0.0-20191108220231-414630eed80f // indirect
github.com/klauspost/compress v1.14.2 // indirect github.com/klauspost/compress v1.15.1 // indirect
github.com/klauspost/cpuid/v2 v2.0.11 // indirect github.com/klauspost/cpuid/v2 v2.0.11 // indirect
github.com/labstack/gommon v0.3.1 // indirect github.com/labstack/gommon v0.3.1 // indirect
github.com/magiconair/properties v1.8.5 // indirect github.com/magiconair/properties v1.8.5 // indirect

22
go.sum
View File

@ -152,8 +152,8 @@ github.com/Rhymen/go-whatsapp v0.1.2-0.20211102134409-31a2e740845c/go.mod h1:DNS
github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo= github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo=
github.com/RoaringBitmap/roaring v0.8.0/go.mod h1:jdT9ykXwHFNdJbEtxePexlFYH9LXucApeS0/+/g+p1I= github.com/RoaringBitmap/roaring v0.8.0/go.mod h1:jdT9ykXwHFNdJbEtxePexlFYH9LXucApeS0/+/g+p1I=
github.com/RoaringBitmap/roaring v0.9.4/go.mod h1:icnadbWcNyfEHlYdr+tDlOTih1Bf/h+rzPpv4sbomAA= github.com/RoaringBitmap/roaring v0.9.4/go.mod h1:icnadbWcNyfEHlYdr+tDlOTih1Bf/h+rzPpv4sbomAA=
github.com/SevereCloud/vksdk/v2 v2.13.1 h1:D11NaP275mW01v2hRF0ycDHdJaIyZEvasZV4MSkg5Sk= github.com/SevereCloud/vksdk/v2 v2.14.0 h1:1lciJC4FWhSQIjjFb3NGyJI7x9sPKk/P6aAvR0ibh1o=
github.com/SevereCloud/vksdk/v2 v2.13.1/go.mod h1:UyOgSj/CYt2dByu3Fyf/y1yT1NoahVi4zECvvrbtPU4= github.com/SevereCloud/vksdk/v2 v2.14.0/go.mod h1:J/iPooVfldjVADo47G5aNxkvlRWAsZnMHpri8sZmck4=
github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0=
github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ=
github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
@ -1008,8 +1008,9 @@ github.com/klauspost/compress v1.13.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8
github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/klauspost/compress v1.14.2 h1:S0OHlFk/Gbon/yauFJ4FfJJF5V0fc5HbBTJazi28pRw=
github.com/klauspost/compress v1.14.2/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.14.2/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/klauspost/compress v1.15.1 h1:y9FcTHGyrebwfP0ZZqFiaxTaiDnUrGkJkI+f583BL1A=
github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
@ -1062,8 +1063,8 @@ github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/lrstanley/girc v0.0.0-20220321215535-9664730c7858 h1:IIbHCRHuANbPoQymk4BWGcRI47RXHOt7bDxPbxQ9rms= github.com/lrstanley/girc v0.0.0-20220409202343-de3f963fb827 h1:BNWTIvCM58QyC78VBgGsMpwnaPsrUw9yadoH3Ge+GiU=
github.com/lrstanley/girc v0.0.0-20220321215535-9664730c7858/go.mod h1:liX5MxHPrwgHaKowoLkYGwbXfYABh1jbZ6FpElbGF1I= github.com/lrstanley/girc v0.0.0-20220409202343-de3f963fb827/go.mod h1:liX5MxHPrwgHaKowoLkYGwbXfYABh1jbZ6FpElbGF1I=
github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
@ -1846,8 +1847,8 @@ golang.org/x/image v0.0.0-20201208152932-35266b937fa6/go.mod h1:FeLwcggjj3mMvU+o
golang.org/x/image v0.0.0-20210216034530-4410531fe030/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20210216034530-4410531fe030/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/image v0.0.0-20210622092929-e6eecd499c2c/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= golang.org/x/image v0.0.0-20210622092929-e6eecd499c2c/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
golang.org/x/image v0.0.0-20220302094943-723b81ca9867 h1:TcHcE0vrmgzNH1v3ppjcMGbhG5+9fMuvOmUYwNEF4q4= golang.org/x/image v0.0.0-20220321031419-a8550c1d254a h1:LnH9RNcpPv5Kzi15lXg42lYMPUf0x8CuPv1YnvBWZAg=
golang.org/x/image v0.0.0-20220302094943-723b81ca9867/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= golang.org/x/image v0.0.0-20220321031419-a8550c1d254a/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20181217174547-8f45f776aaf1/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181217174547-8f45f776aaf1/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@ -2446,8 +2447,9 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk=
@ -2694,8 +2696,8 @@ modernc.org/opt v0.1.1/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
modernc.org/ql v1.0.0/go.mod h1:xGVyrLIatPcO2C1JvI/Co8c0sr6y91HKFNy4pt9JXEY= modernc.org/ql v1.0.0/go.mod h1:xGVyrLIatPcO2C1JvI/Co8c0sr6y91HKFNy4pt9JXEY=
modernc.org/sortutil v1.1.0/go.mod h1:ZyL98OQHJgH9IEfN71VsamvJgrtRX9Dj2gX+vH86L1k= modernc.org/sortutil v1.1.0/go.mod h1:ZyL98OQHJgH9IEfN71VsamvJgrtRX9Dj2gX+vH86L1k=
modernc.org/sqlite v1.10.6/go.mod h1:Z9FEjUtZP4qFEg6/SiADg9XCER7aYy9a/j7Pg9P7CPs= modernc.org/sqlite v1.10.6/go.mod h1:Z9FEjUtZP4qFEg6/SiADg9XCER7aYy9a/j7Pg9P7CPs=
modernc.org/sqlite v1.15.4 h1:pr3EA3Rety3j1c/9pCyGAe5d3vjF6wQwusHdgGCjIqc= modernc.org/sqlite v1.16.0 h1:DdvOGaWN0y+X7t2L7RUD63gcwbVjYZjcBZnA68g44EI=
modernc.org/sqlite v1.15.4/go.mod h1:Jwe13ItpESZ+78K5WS6+AjXsUg+JvirsjN3iIDO4C8k= modernc.org/sqlite v1.16.0/go.mod h1:Jwe13ItpESZ+78K5WS6+AjXsUg+JvirsjN3iIDO4C8k=
modernc.org/strutil v1.1.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= modernc.org/strutil v1.1.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs=
modernc.org/strutil v1.1.1 h1:xv+J1BXY3Opl2ALrBwyfEikFAj8pmqcpnfmuwUwcozs= modernc.org/strutil v1.1.1 h1:xv+J1BXY3Opl2ALrBwyfEikFAj8pmqcpnfmuwUwcozs=
modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw= modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw=

2
vendor/github.com/SevereCloud/vksdk/v2/README.md generated vendored
View File

@ -1,7 +1,7 @@
# VK SDK for Golang # VK SDK for Golang
[![PkgGoDev](https://pkg.go.dev/badge/github.com/SevereCloud/vksdk/v2/v2)](https://pkg.go.dev/github.com/SevereCloud/vksdk/v2?tab=subdirectories) [![PkgGoDev](https://pkg.go.dev/badge/github.com/SevereCloud/vksdk/v2/v2)](https://pkg.go.dev/github.com/SevereCloud/vksdk/v2?tab=subdirectories)
[![VK Developers](https://img.shields.io/badge/developers-%234a76a8.svg?logo=VK&logoColor=white)](https://vk.com/dev/) [![VK Developers](https://img.shields.io/badge/developers-%234a76a8.svg?logo=VK&logoColor=white)](https://dev.vk.com/)
[![codecov](https://codecov.io/gh/SevereCloud/vksdk/branch/master/graph/badge.svg)](https://codecov.io/gh/SevereCloud/vksdk) [![codecov](https://codecov.io/gh/SevereCloud/vksdk/branch/master/graph/badge.svg)](https://codecov.io/gh/SevereCloud/vksdk)
[![VK chat](https://img.shields.io/badge/VK%20chat-%234a76a8.svg?logo=VK&logoColor=white)](https://vk.me/join/AJQ1d6Or8Q00Y_CSOESfbqGt) [![VK chat](https://img.shields.io/badge/VK%20chat-%234a76a8.svg?logo=VK&logoColor=white)](https://vk.me/join/AJQ1d6Or8Q00Y_CSOESfbqGt)
[![release](https://img.shields.io/github/v/tag/SevereCloud/vksdk?label=release)](https://github.com/SevereCloud/vksdk/releases) [![release](https://img.shields.io/github/v/tag/SevereCloud/vksdk?label=release)](https://github.com/SevereCloud/vksdk/releases)

2
vendor/github.com/SevereCloud/vksdk/v2/api/account.go generated vendored
View File

@ -141,6 +141,8 @@ func (vk *VK) AccountSetInfo(params Params) (response int, err error) {
// AccountSetNameInMenu sets an application screen name // AccountSetNameInMenu sets an application screen name
// (up to 17 characters), that is shown to the user in the left menu. // (up to 17 characters), that is shown to the user in the left menu.
// //
// Deprecated: This method is deprecated and may be disabled soon, please avoid
//
// https://vk.com/dev/account.setNameInMenu // https://vk.com/dev/account.setNameInMenu
func (vk *VK) AccountSetNameInMenu(params Params) (response int, err error) { func (vk *VK) AccountSetNameInMenu(params Params) (response int, err error) {
err = vk.RequestUnmarshal("account.setNameInMenu", &response, params) err = vk.RequestUnmarshal("account.setNameInMenu", &response, params)

2
vendor/github.com/SevereCloud/vksdk/v2/api/store.go generated vendored
View File

@ -1,4 +1,4 @@
package api // import "github.com/SevereCloud/vksdk/api" package api // import "github.com/SevereCloud/vksdk/v2/api"
import ( import (
"github.com/SevereCloud/vksdk/v2/object" "github.com/SevereCloud/vksdk/v2/object"

35
vendor/github.com/SevereCloud/vksdk/v2/api/video.go generated vendored
View File

@ -235,6 +235,17 @@ func (vk *VK) VideoGetCommentsExtended(params Params) (response VideoGetComments
return return
} }
// VideoLiveGetCategoriesResponse struct.
type VideoLiveGetCategoriesResponse []object.VideoLiveCategory
// VideoLiveGetCategories method.
//
// https://vk.com/dev/video.liveGetCategories
func (vk *VK) VideoLiveGetCategories(params Params) (response VideoLiveGetCategoriesResponse, err error) {
err = vk.RequestUnmarshal("video.liveGetCategories", &response, params)
return
}
// VideoRemoveFromAlbum allows you to remove the video from the album. // VideoRemoveFromAlbum allows you to remove the video from the album.
// //
// https://vk.com/dev/video.removeFromAlbum // https://vk.com/dev/video.removeFromAlbum
@ -336,3 +347,27 @@ func (vk *VK) VideoSearchExtended(params Params) (response VideoSearchExtendedRe
return return
} }
// VideoStartStreamingResponse struct.
type VideoStartStreamingResponse object.VideoLive
// VideoStartStreaming method.
//
// https://vk.com/dev/video.startStreaming
func (vk *VK) VideoStartStreaming(params Params) (response VideoStartStreamingResponse, err error) {
err = vk.RequestUnmarshal("video.startStreaming", &response, params)
return
}
// VideoStopStreamingResponse struct.
type VideoStopStreamingResponse struct {
UniqueViewers int `json:"unique_viewers"`
}
// VideoStopStreaming method.
//
// https://vk.com/dev/video.stopStreaming
func (vk *VK) VideoStopStreaming(params Params) (response VideoStopStreamingResponse, err error) {
err = vk.RequestUnmarshal("video.stopStreaming", &response, params)
return
}

2
vendor/github.com/SevereCloud/vksdk/v2/doc.go generated vendored
View File

@ -7,6 +7,6 @@ package vksdk
// Module constants. // Module constants.
const ( const (
Version = "2.13.1" Version = "2.14.0"
API = "5.131" API = "5.131"
) )

5
vendor/github.com/SevereCloud/vksdk/v2/events/context.go generated vendored
View File

@ -15,3 +15,8 @@ func GroupIDFromContext(ctx context.Context) int {
func EventIDFromContext(ctx context.Context) string { func EventIDFromContext(ctx context.Context) string {
return ctx.Value(internal.EventIDKey).(string) return ctx.Value(internal.EventIDKey).(string)
} }
// VersionFromContext returns the version from context.
func VersionFromContext(ctx context.Context) string {
return ctx.Value(internal.EventVersionKey).(string)
}

2
vendor/github.com/SevereCloud/vksdk/v2/events/events.go generated vendored
View File

@ -81,6 +81,7 @@ type GroupEvent struct {
Object json.RawMessage `json:"object"` Object json.RawMessage `json:"object"`
GroupID int `json:"group_id"` GroupID int `json:"group_id"`
EventID string `json:"event_id"` EventID string `json:"event_id"`
V string `json:"v"`
Secret string `json:"secret"` Secret string `json:"secret"`
} }
@ -158,6 +159,7 @@ func NewFuncList() *FuncList {
func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:gocyclo func (fl FuncList) Handler(ctx context.Context, e GroupEvent) error { // nolint:gocyclo
ctx = context.WithValue(ctx, internal.GroupIDKey, e.GroupID) ctx = context.WithValue(ctx, internal.GroupIDKey, e.GroupID)
ctx = context.WithValue(ctx, internal.EventIDKey, e.EventID) ctx = context.WithValue(ctx, internal.EventIDKey, e.EventID)
ctx = context.WithValue(ctx, internal.EventVersionKey, e.V)
if sliceFunc, ok := fl.special[e.Type]; ok { if sliceFunc, ok := fl.special[e.Type]; ok {
for _, f := range sliceFunc { for _, f := range sliceFunc {

1
vendor/github.com/SevereCloud/vksdk/v2/internal/transport.go generated vendored
View File

@ -28,6 +28,7 @@ const (
CallbackRetryCounterKey CallbackRetryCounterKey
CallbackRetryAfterKey CallbackRetryAfterKey
CallbackRemove CallbackRemove
EventVersionKey
) )
// ContextClient return *http.Client. // ContextClient return *http.Client.

1
vendor/github.com/SevereCloud/vksdk/v2/object/account.go generated vendored
View File

@ -84,7 +84,6 @@ type AccountAccountCounters struct {
// AccountInfo struct. // AccountInfo struct.
type AccountInfo struct { type AccountInfo struct {
// Country code. // Country code.
Country string `json:"country"` Country string `json:"country"`

1
vendor/github.com/SevereCloud/vksdk/v2/object/messages.go generated vendored
View File

@ -409,7 +409,6 @@ type MessageContentSource struct {
Type string `json:"type"` Type string `json:"type"`
MessageContentSourceMessage // type message MessageContentSourceMessage // type message
MessageContentSourceURL // type url MessageContentSourceURL // type url
} }
// NewMessageContentSourceMessage ... // NewMessageContentSourceMessage ...

5
vendor/github.com/SevereCloud/vksdk/v2/object/users.go generated vendored
View File

@ -77,10 +77,13 @@ type UsersUser struct {
Trending BaseBoolInt `json:"trending"` Trending BaseBoolInt `json:"trending"`
Blacklisted BaseBoolInt `json:"blacklisted"` Blacklisted BaseBoolInt `json:"blacklisted"`
BlacklistedByMe BaseBoolInt `json:"blacklisted_by_me"` BlacklistedByMe BaseBoolInt `json:"blacklisted_by_me"`
// Deprecated: Facebook и Instagram запрещены в России, Meta признана экстремистской организацией...
Facebook string `json:"facebook"` Facebook string `json:"facebook"`
// Deprecated: Facebook и Instagram запрещены в России, Meta признана экстремистской организацией...
FacebookName string `json:"facebook_name"` FacebookName string `json:"facebook_name"`
Twitter string `json:"twitter"` // Deprecated: Facebook и Instagram запрещены в России, Meta признана экстремистской организацией...
Instagram string `json:"instagram"` Instagram string `json:"instagram"`
Twitter string `json:"twitter"`
Site string `json:"site"` Site string `json:"site"`
Status string `json:"status"` Status string `json:"status"`
StatusAudio AudioAudio `json:"status_audio"` StatusAudio AudioAudio `json:"status_audio"`

24
vendor/github.com/SevereCloud/vksdk/v2/object/video.go generated vendored
View File

@ -297,3 +297,27 @@ type VideoVideoImage struct {
BaseImage BaseImage
WithPadding BaseBoolInt `json:"with_padding"` WithPadding BaseBoolInt `json:"with_padding"`
} }
// VideoLive struct.
type VideoLive struct {
OwnerID int `json:"owner_id"`
VideoID int `json:"video_id"`
Name string `json:"name"`
Description string `json:"description"`
AccessKey string `json:"access_key"`
Stream VideoLiveStream `json:"stream"`
}
// VideoLiveStream struct.
type VideoLiveStream struct {
URL string `json:"url"`
Key string `json:"key"`
OKMPURL string `json:"okmp_url"`
}
// VideoLiveCategory struct.
type VideoLiveCategory struct {
ID int `json:"id"`
Label string `json:"label"`
Sublist []VideoLiveCategory `json:"sublist,omitempty"`
}

51
vendor/github.com/klauspost/compress/README.md generated vendored
View File

@ -17,6 +17,42 @@ This package provides various compression algorithms.
# changelog # changelog
* Mar 3, 2022 (v1.15.0)
* zstd: Refactor decoder by @klauspost in [#498](https://github.com/klauspost/compress/pull/498)
* zstd: Add stream encoding without goroutines by @klauspost in [#505](https://github.com/klauspost/compress/pull/505)
* huff0: Prevent single blocks exceeding 16 bits by @klauspost in[#507](https://github.com/klauspost/compress/pull/507)
* flate: Inline literal emission by @klauspost in [#509](https://github.com/klauspost/compress/pull/509)
* gzhttp: Add zstd to transport by @klauspost in [#400](https://github.com/klauspost/compress/pull/400)
* gzhttp: Make content-type optional by @klauspost in [#510](https://github.com/klauspost/compress/pull/510)
<details>
<summary>See Details</summary>
Both compression and decompression now supports "synchronous" stream operations. This means that whenever "concurrency" is set to 1, they will operate without spawning goroutines.
Stream decompression is now faster on asynchronous, since the goroutine allocation much more effectively splits the workload. On typical streams this will typically use 2 cores fully for decompression. When a stream has finished decoding no goroutines will be left over, so decoders can now safely be pooled and still be garbage collected.
While the release has been extensively tested, it is recommended to testing when upgrading.
</details>
* Feb 22, 2022 (v1.14.4)
* flate: Fix rare huffman only (-2) corruption. [#503](https://github.com/klauspost/compress/pull/503)
* zip: Update deprecated CreateHeaderRaw to correctly call CreateRaw by @saracen in [#502](https://github.com/klauspost/compress/pull/502)
* zip: don't read data descriptor early by @saracen in [#501](https://github.com/klauspost/compress/pull/501) #501
* huff0: Use static decompression buffer up to 30% faster by @klauspost in [#499](https://github.com/klauspost/compress/pull/499) [#500](https://github.com/klauspost/compress/pull/500)
* Feb 17, 2022 (v1.14.3)
* flate: Improve fastest levels compression speed ~10% more throughput. [#482](https://github.com/klauspost/compress/pull/482) [#489](https://github.com/klauspost/compress/pull/489) [#490](https://github.com/klauspost/compress/pull/490) [#491](https://github.com/klauspost/compress/pull/491) [#494](https://github.com/klauspost/compress/pull/494) [#478](https://github.com/klauspost/compress/pull/478)
* flate: Faster decompression speed, ~5-10%. [#483](https://github.com/klauspost/compress/pull/483)
* s2: Faster compression with Go v1.18 and amd64 microarch level 3+. [#484](https://github.com/klauspost/compress/pull/484) [#486](https://github.com/klauspost/compress/pull/486)
* Jan 25, 2022 (v1.14.2)
* zstd: improve header decoder by @dsnet [#476](https://github.com/klauspost/compress/pull/476)
* zstd: Add bigger default blocks [#469](https://github.com/klauspost/compress/pull/469)
* zstd: Remove unused decompression buffer [#470](https://github.com/klauspost/compress/pull/470)
* zstd: Fix logically dead code by @ningmingxiao [#472](https://github.com/klauspost/compress/pull/472)
* flate: Improve level 7-9 [#471](https://github.com/klauspost/compress/pull/471) [#473](https://github.com/klauspost/compress/pull/473)
* zstd: Add noasm tag for xxhash [#475](https://github.com/klauspost/compress/pull/475)
* Jan 11, 2022 (v1.14.1) * Jan 11, 2022 (v1.14.1)
* s2: Add stream index in [#462](https://github.com/klauspost/compress/pull/462) * s2: Add stream index in [#462](https://github.com/klauspost/compress/pull/462)
* flate: Speed and efficiency improvements in [#439](https://github.com/klauspost/compress/pull/439) [#461](https://github.com/klauspost/compress/pull/461) [#455](https://github.com/klauspost/compress/pull/455) [#452](https://github.com/klauspost/compress/pull/452) [#458](https://github.com/klauspost/compress/pull/458) * flate: Speed and efficiency improvements in [#439](https://github.com/klauspost/compress/pull/439) [#461](https://github.com/klauspost/compress/pull/461) [#455](https://github.com/klauspost/compress/pull/455) [#452](https://github.com/klauspost/compress/pull/452) [#458](https://github.com/klauspost/compress/pull/458)
@ -53,6 +89,9 @@ This package provides various compression algorithms.
* zstd: Detect short invalid signatures [#382](https://github.com/klauspost/compress/pull/382) * zstd: Detect short invalid signatures [#382](https://github.com/klauspost/compress/pull/382)
* zstd: Spawn decoder goroutine only if needed. [#380](https://github.com/klauspost/compress/pull/380) * zstd: Spawn decoder goroutine only if needed. [#380](https://github.com/klauspost/compress/pull/380)
<details>
<summary>See changes to v1.12.x</summary>
* May 25, 2021 (v1.12.3) * May 25, 2021 (v1.12.3)
* deflate: Better/faster Huffman encoding [#374](https://github.com/klauspost/compress/pull/374) * deflate: Better/faster Huffman encoding [#374](https://github.com/klauspost/compress/pull/374)
* deflate: Allocate less for history. [#375](https://github.com/klauspost/compress/pull/375) * deflate: Allocate less for history. [#375](https://github.com/klauspost/compress/pull/375)
@ -74,9 +113,10 @@ This package provides various compression algorithms.
* s2c/s2d/s2sx: Always truncate when writing files [#352](https://github.com/klauspost/compress/pull/352) * s2c/s2d/s2sx: Always truncate when writing files [#352](https://github.com/klauspost/compress/pull/352)
* zstd: Reduce memory usage further when using [WithLowerEncoderMem](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithLowerEncoderMem) [#346](https://github.com/klauspost/compress/pull/346) * zstd: Reduce memory usage further when using [WithLowerEncoderMem](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithLowerEncoderMem) [#346](https://github.com/klauspost/compress/pull/346)
* s2: Fix potential problem with amd64 assembly and profilers [#349](https://github.com/klauspost/compress/pull/349) * s2: Fix potential problem with amd64 assembly and profilers [#349](https://github.com/klauspost/compress/pull/349)
</details>
<details> <details>
<summary>See changes prior to v1.12.1</summary> <summary>See changes to v1.11.x</summary>
* Mar 26, 2021 (v1.11.13) * Mar 26, 2021 (v1.11.13)
* zstd: Big speedup on small dictionary encodes [#344](https://github.com/klauspost/compress/pull/344) [#345](https://github.com/klauspost/compress/pull/345) * zstd: Big speedup on small dictionary encodes [#344](https://github.com/klauspost/compress/pull/344) [#345](https://github.com/klauspost/compress/pull/345)
@ -135,7 +175,7 @@ This package provides various compression algorithms.
</details> </details>
<details> <details>
<summary>See changes prior to v1.11.0</summary> <summary>See changes to v1.10.x</summary>
* July 8, 2020 (v1.10.11) * July 8, 2020 (v1.10.11)
* zstd: Fix extra block when compressing with ReadFrom. [#278](https://github.com/klauspost/compress/pull/278) * zstd: Fix extra block when compressing with ReadFrom. [#278](https://github.com/klauspost/compress/pull/278)
@ -297,11 +337,6 @@ This package provides various compression algorithms.
# deflate usage # deflate usage
* [High Throughput Benchmark](http://blog.klauspost.com/go-gzipdeflate-benchmarks/).
* [Small Payload/Webserver Benchmarks](http://blog.klauspost.com/gzip-performance-for-go-webservers/).
* [Linear Time Compression](http://blog.klauspost.com/constant-time-gzipzip-compression/).
* [Re-balancing Deflate Compression Levels](https://blog.klauspost.com/rebalancing-deflate-compression-levels/)
The packages are drop-in replacements for standard libraries. Simply replace the import path to use them: The packages are drop-in replacements for standard libraries. Simply replace the import path to use them:
| old import | new import | Documentation | old import | new import | Documentation
@ -323,6 +358,8 @@ Memory usage is typically 1MB for a Writer. stdlib is in the same range.
If you expect to have a lot of concurrently allocated Writers consider using If you expect to have a lot of concurrently allocated Writers consider using
the stateless compress described below. the stateless compress described below.
For compression performance, see: [this spreadsheet](https://docs.google.com/spreadsheets/d/1nuNE2nPfuINCZJRMt6wFWhKpToF95I47XjSsc-1rbPQ/edit?usp=sharing).
# Stateless compression # Stateless compression
This package offers stateless compression as a special option for gzip/deflate. This package offers stateless compression as a special option for gzip/deflate.

5
vendor/github.com/klauspost/compress/huff0/autogen.go generated vendored Normal file
View File

@ -0,0 +1,5 @@
package huff0
//go:generate go run generate.go
//go:generate asmfmt -w decompress_amd64.s
//go:generate asmfmt -w decompress_8b_amd64.s

126
vendor/github.com/klauspost/compress/huff0/bitreader.go generated vendored
View File

@ -8,115 +8,10 @@ package huff0
import ( import (
"encoding/binary" "encoding/binary"
"errors" "errors"
"fmt"
"io" "io"
) )
// bitReader reads a bitstream in reverse.
// The last set bit indicates the start of the stream and is used
// for aligning the input.
type bitReader struct {
in []byte
off uint // next byte to read is at in[off - 1]
value uint64
bitsRead uint8
}
// init initializes and resets the bit reader.
func (b *bitReader) init(in []byte) error {
if len(in) < 1 {
return errors.New("corrupt stream: too short")
}
b.in = in
b.off = uint(len(in))
// The highest bit of the last byte indicates where to start
v := in[len(in)-1]
if v == 0 {
return errors.New("corrupt stream, did not find end of stream")
}
b.bitsRead = 64
b.value = 0
if len(in) >= 8 {
b.fillFastStart()
} else {
b.fill()
b.fill()
}
b.bitsRead += 8 - uint8(highBit32(uint32(v)))
return nil
}
// peekBitsFast requires that at least one bit is requested every time.
// There are no checks if the buffer is filled.
func (b *bitReader) peekBitsFast(n uint8) uint16 {
const regMask = 64 - 1
v := uint16((b.value << (b.bitsRead & regMask)) >> ((regMask + 1 - n) & regMask))
return v
}
// fillFast() will make sure at least 32 bits are available.
// There must be at least 4 bytes available.
func (b *bitReader) fillFast() {
if b.bitsRead < 32 {
return
}
// 2 bounds checks.
v := b.in[b.off-4 : b.off]
v = v[:4]
low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24)
b.value = (b.value << 32) | uint64(low)
b.bitsRead -= 32
b.off -= 4
}
func (b *bitReader) advance(n uint8) {
b.bitsRead += n
}
// fillFastStart() assumes the bitreader is empty and there is at least 8 bytes to read.
func (b *bitReader) fillFastStart() {
// Do single re-slice to avoid bounds checks.
b.value = binary.LittleEndian.Uint64(b.in[b.off-8:])
b.bitsRead = 0
b.off -= 8
}
// fill() will make sure at least 32 bits are available.
func (b *bitReader) fill() {
if b.bitsRead < 32 {
return
}
if b.off > 4 {
v := b.in[b.off-4:]
v = v[:4]
low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24)
b.value = (b.value << 32) | uint64(low)
b.bitsRead -= 32
b.off -= 4
return
}
for b.off > 0 {
b.value = (b.value << 8) | uint64(b.in[b.off-1])
b.bitsRead -= 8
b.off--
}
}
// finished returns true if all bits have been read from the bit stream.
func (b *bitReader) finished() bool {
return b.off == 0 && b.bitsRead >= 64
}
// close the bitstream and returns an error if out-of-buffer reads occurred.
func (b *bitReader) close() error {
// Release reference.
b.in = nil
if b.bitsRead > 64 {
return io.ErrUnexpectedEOF
}
return nil
}
// bitReader reads a bitstream in reverse. // bitReader reads a bitstream in reverse.
// The last set bit indicates the start of the stream and is used // The last set bit indicates the start of the stream and is used
// for aligning the input. // for aligning the input.
@ -213,10 +108,17 @@ func (b *bitReaderBytes) finished() bool {
return b.off == 0 && b.bitsRead >= 64 return b.off == 0 && b.bitsRead >= 64
} }
func (b *bitReaderBytes) remaining() uint {
return b.off*8 + uint(64-b.bitsRead)
}
// close the bitstream and returns an error if out-of-buffer reads occurred. // close the bitstream and returns an error if out-of-buffer reads occurred.
func (b *bitReaderBytes) close() error { func (b *bitReaderBytes) close() error {
// Release reference. // Release reference.
b.in = nil b.in = nil
if b.remaining() > 0 {
return fmt.Errorf("corrupt input: %d bits remain on stream", b.remaining())
}
if b.bitsRead > 64 { if b.bitsRead > 64 {
return io.ErrUnexpectedEOF return io.ErrUnexpectedEOF
} }
@ -263,6 +165,11 @@ func (b *bitReaderShifted) peekBitsFast(n uint8) uint16 {
return uint16(b.value >> ((64 - n) & 63)) return uint16(b.value >> ((64 - n) & 63))
} }
// peekTopBits(n) is equvialent to peekBitFast(64 - n)
func (b *bitReaderShifted) peekTopBits(n uint8) uint16 {
return uint16(b.value >> n)
}
func (b *bitReaderShifted) advance(n uint8) { func (b *bitReaderShifted) advance(n uint8) {
b.bitsRead += n b.bitsRead += n
b.value <<= n & 63 b.value <<= n & 63
@ -318,10 +225,17 @@ func (b *bitReaderShifted) finished() bool {
return b.off == 0 && b.bitsRead >= 64 return b.off == 0 && b.bitsRead >= 64
} }
func (b *bitReaderShifted) remaining() uint {
return b.off*8 + uint(64-b.bitsRead)
}
// close the bitstream and returns an error if out-of-buffer reads occurred. // close the bitstream and returns an error if out-of-buffer reads occurred.
func (b *bitReaderShifted) close() error { func (b *bitReaderShifted) close() error {
// Release reference. // Release reference.
b.in = nil b.in = nil
if b.remaining() > 0 {
return fmt.Errorf("corrupt input: %d bits remain on stream", b.remaining())
}
if b.bitsRead > 64 { if b.bitsRead > 64 {
return io.ErrUnexpectedEOF return io.ErrUnexpectedEOF
} }

9
vendor/github.com/klauspost/compress/huff0/compress.go generated vendored
View File

@ -2,6 +2,7 @@ package huff0
import ( import (
"fmt" "fmt"
"math"
"runtime" "runtime"
"sync" "sync"
) )
@ -289,6 +290,10 @@ func (s *Scratch) compress4X(src []byte) ([]byte, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
if len(s.Out)-idx > math.MaxUint16 {
// We cannot store the size in the jump table
return nil, ErrIncompressible
}
// Write compressed length as little endian before block. // Write compressed length as little endian before block.
if i < 3 { if i < 3 {
// Last length is not written. // Last length is not written.
@ -332,6 +337,10 @@ func (s *Scratch) compress4Xp(src []byte) ([]byte, error) {
return nil, errs[i] return nil, errs[i]
} }
o := s.tmpOut[i] o := s.tmpOut[i]
if len(o) > math.MaxUint16 {
// We cannot store the size in the jump table
return nil, ErrIncompressible
}
// Write compressed length as little endian before block. // Write compressed length as little endian before block.
if i < 3 { if i < 3 {
// Last length is not written. // Last length is not written.

493
vendor/github.com/klauspost/compress/huff0/decompress.go generated vendored
View File

@ -4,6 +4,7 @@ import (
"errors" "errors"
"fmt" "fmt"
"io" "io"
"sync"
"github.com/klauspost/compress/fse" "github.com/klauspost/compress/fse"
) )
@ -216,6 +217,7 @@ func (s *Scratch) Decoder() *Decoder {
return &Decoder{ return &Decoder{
dt: s.dt, dt: s.dt,
actualTableLog: s.actualTableLog, actualTableLog: s.actualTableLog,
bufs: &s.decPool,
} }
} }
@ -223,6 +225,15 @@ func (s *Scratch) Decoder() *Decoder {
type Decoder struct { type Decoder struct {
dt dTable dt dTable
actualTableLog uint8 actualTableLog uint8
bufs *sync.Pool
}
func (d *Decoder) buffer() *[4][256]byte {
buf, ok := d.bufs.Get().(*[4][256]byte)
if ok {
return buf
}
return &[4][256]byte{}
} }
// Decompress1X will decompress a 1X encoded stream. // Decompress1X will decompress a 1X encoded stream.
@ -249,7 +260,8 @@ func (d *Decoder) Decompress1X(dst, src []byte) ([]byte, error) {
dt := d.dt.single[:tlSize] dt := d.dt.single[:tlSize]
// Use temp table to avoid bound checks/append penalty. // Use temp table to avoid bound checks/append penalty.
var buf [256]byte bufs := d.buffer()
buf := &bufs[0]
var off uint8 var off uint8
for br.off >= 8 { for br.off >= 8 {
@ -277,6 +289,7 @@ func (d *Decoder) Decompress1X(dst, src []byte) ([]byte, error) {
if off == 0 { if off == 0 {
if len(dst)+256 > maxDecodedSize { if len(dst)+256 > maxDecodedSize {
br.close() br.close()
d.bufs.Put(bufs)
return nil, ErrMaxDecodedSizeExceeded return nil, ErrMaxDecodedSizeExceeded
} }
dst = append(dst, buf[:]...) dst = append(dst, buf[:]...)
@ -284,6 +297,7 @@ func (d *Decoder) Decompress1X(dst, src []byte) ([]byte, error) {
} }
if len(dst)+int(off) > maxDecodedSize { if len(dst)+int(off) > maxDecodedSize {
d.bufs.Put(bufs)
br.close() br.close()
return nil, ErrMaxDecodedSizeExceeded return nil, ErrMaxDecodedSizeExceeded
} }
@ -310,6 +324,7 @@ func (d *Decoder) Decompress1X(dst, src []byte) ([]byte, error) {
} }
} }
if len(dst) >= maxDecodedSize { if len(dst) >= maxDecodedSize {
d.bufs.Put(bufs)
br.close() br.close()
return nil, ErrMaxDecodedSizeExceeded return nil, ErrMaxDecodedSizeExceeded
} }
@ -319,6 +334,7 @@ func (d *Decoder) Decompress1X(dst, src []byte) ([]byte, error) {
bitsLeft -= nBits bitsLeft -= nBits
dst = append(dst, uint8(v.entry>>8)) dst = append(dst, uint8(v.entry>>8))
} }
d.bufs.Put(bufs)
return dst, br.close() return dst, br.close()
} }
@ -341,7 +357,8 @@ func (d *Decoder) decompress1X8Bit(dst, src []byte) ([]byte, error) {
dt := d.dt.single[:256] dt := d.dt.single[:256]
// Use temp table to avoid bound checks/append penalty. // Use temp table to avoid bound checks/append penalty.
var buf [256]byte bufs := d.buffer()
buf := &bufs[0]
var off uint8 var off uint8
switch d.actualTableLog { switch d.actualTableLog {
@ -369,6 +386,7 @@ func (d *Decoder) decompress1X8Bit(dst, src []byte) ([]byte, error) {
if off == 0 { if off == 0 {
if len(dst)+256 > maxDecodedSize { if len(dst)+256 > maxDecodedSize {
br.close() br.close()
d.bufs.Put(bufs)
return nil, ErrMaxDecodedSizeExceeded return nil, ErrMaxDecodedSizeExceeded
} }
dst = append(dst, buf[:]...) dst = append(dst, buf[:]...)
@ -398,6 +416,7 @@ func (d *Decoder) decompress1X8Bit(dst, src []byte) ([]byte, error) {
if off == 0 { if off == 0 {
if len(dst)+256 > maxDecodedSize { if len(dst)+256 > maxDecodedSize {
br.close() br.close()
d.bufs.Put(bufs)
return nil, ErrMaxDecodedSizeExceeded return nil, ErrMaxDecodedSizeExceeded
} }
dst = append(dst, buf[:]...) dst = append(dst, buf[:]...)
@ -426,6 +445,7 @@ func (d *Decoder) decompress1X8Bit(dst, src []byte) ([]byte, error) {
off += 4 off += 4
if off == 0 { if off == 0 {
if len(dst)+256 > maxDecodedSize { if len(dst)+256 > maxDecodedSize {
d.bufs.Put(bufs)
br.close() br.close()
return nil, ErrMaxDecodedSizeExceeded return nil, ErrMaxDecodedSizeExceeded
} }
@ -455,6 +475,7 @@ func (d *Decoder) decompress1X8Bit(dst, src []byte) ([]byte, error) {
off += 4 off += 4
if off == 0 { if off == 0 {
if len(dst)+256 > maxDecodedSize { if len(dst)+256 > maxDecodedSize {
d.bufs.Put(bufs)
br.close() br.close()
return nil, ErrMaxDecodedSizeExceeded return nil, ErrMaxDecodedSizeExceeded
} }
@ -484,6 +505,7 @@ func (d *Decoder) decompress1X8Bit(dst, src []byte) ([]byte, error) {
off += 4 off += 4
if off == 0 { if off == 0 {
if len(dst)+256 > maxDecodedSize { if len(dst)+256 > maxDecodedSize {
d.bufs.Put(bufs)
br.close() br.close()
return nil, ErrMaxDecodedSizeExceeded return nil, ErrMaxDecodedSizeExceeded
} }
@ -513,6 +535,7 @@ func (d *Decoder) decompress1X8Bit(dst, src []byte) ([]byte, error) {
off += 4 off += 4
if off == 0 { if off == 0 {
if len(dst)+256 > maxDecodedSize { if len(dst)+256 > maxDecodedSize {
d.bufs.Put(bufs)
br.close() br.close()
return nil, ErrMaxDecodedSizeExceeded return nil, ErrMaxDecodedSizeExceeded
} }
@ -542,6 +565,7 @@ func (d *Decoder) decompress1X8Bit(dst, src []byte) ([]byte, error) {
off += 4 off += 4
if off == 0 { if off == 0 {
if len(dst)+256 > maxDecodedSize { if len(dst)+256 > maxDecodedSize {
d.bufs.Put(bufs)
br.close() br.close()
return nil, ErrMaxDecodedSizeExceeded return nil, ErrMaxDecodedSizeExceeded
} }
@ -571,6 +595,7 @@ func (d *Decoder) decompress1X8Bit(dst, src []byte) ([]byte, error) {
off += 4 off += 4
if off == 0 { if off == 0 {
if len(dst)+256 > maxDecodedSize { if len(dst)+256 > maxDecodedSize {
d.bufs.Put(bufs)
br.close() br.close()
return nil, ErrMaxDecodedSizeExceeded return nil, ErrMaxDecodedSizeExceeded
} }
@ -578,10 +603,12 @@ func (d *Decoder) decompress1X8Bit(dst, src []byte) ([]byte, error) {
} }
} }
default: default:
d.bufs.Put(bufs)
return nil, fmt.Errorf("invalid tablelog: %d", d.actualTableLog) return nil, fmt.Errorf("invalid tablelog: %d", d.actualTableLog)
} }
if len(dst)+int(off) > maxDecodedSize { if len(dst)+int(off) > maxDecodedSize {
d.bufs.Put(bufs)
br.close() br.close()
return nil, ErrMaxDecodedSizeExceeded return nil, ErrMaxDecodedSizeExceeded
} }
@ -601,6 +628,7 @@ func (d *Decoder) decompress1X8Bit(dst, src []byte) ([]byte, error) {
} }
if len(dst) >= maxDecodedSize { if len(dst) >= maxDecodedSize {
br.close() br.close()
d.bufs.Put(bufs)
return nil, ErrMaxDecodedSizeExceeded return nil, ErrMaxDecodedSizeExceeded
} }
v := dt[br.peekByteFast()>>shift] v := dt[br.peekByteFast()>>shift]
@ -609,6 +637,7 @@ func (d *Decoder) decompress1X8Bit(dst, src []byte) ([]byte, error) {
bitsLeft -= int8(nBits) bitsLeft -= int8(nBits)
dst = append(dst, uint8(v.entry>>8)) dst = append(dst, uint8(v.entry>>8))
} }
d.bufs.Put(bufs)
return dst, br.close() return dst, br.close()
} }
@ -628,7 +657,8 @@ func (d *Decoder) decompress1X8BitExactly(dst, src []byte) ([]byte, error) {
dt := d.dt.single[:256] dt := d.dt.single[:256]
// Use temp table to avoid bound checks/append penalty. // Use temp table to avoid bound checks/append penalty.
var buf [256]byte bufs := d.buffer()
buf := &bufs[0]
var off uint8 var off uint8
const shift = 56 const shift = 56
@ -655,6 +685,7 @@ func (d *Decoder) decompress1X8BitExactly(dst, src []byte) ([]byte, error) {
off += 4 off += 4
if off == 0 { if off == 0 {
if len(dst)+256 > maxDecodedSize { if len(dst)+256 > maxDecodedSize {
d.bufs.Put(bufs)
br.close() br.close()
return nil, ErrMaxDecodedSizeExceeded return nil, ErrMaxDecodedSizeExceeded
} }
@ -663,6 +694,7 @@ func (d *Decoder) decompress1X8BitExactly(dst, src []byte) ([]byte, error) {
} }
if len(dst)+int(off) > maxDecodedSize { if len(dst)+int(off) > maxDecodedSize {
d.bufs.Put(bufs)
br.close() br.close()
return nil, ErrMaxDecodedSizeExceeded return nil, ErrMaxDecodedSizeExceeded
} }
@ -679,6 +711,7 @@ func (d *Decoder) decompress1X8BitExactly(dst, src []byte) ([]byte, error) {
} }
} }
if len(dst) >= maxDecodedSize { if len(dst) >= maxDecodedSize {
d.bufs.Put(bufs)
br.close() br.close()
return nil, ErrMaxDecodedSizeExceeded return nil, ErrMaxDecodedSizeExceeded
} }
@ -688,195 +721,10 @@ func (d *Decoder) decompress1X8BitExactly(dst, src []byte) ([]byte, error) {
bitsLeft -= int8(nBits) bitsLeft -= int8(nBits)
dst = append(dst, uint8(v.entry>>8)) dst = append(dst, uint8(v.entry>>8))
} }
d.bufs.Put(bufs)
return dst, br.close() return dst, br.close()
} }
// Decompress4X will decompress a 4X encoded stream.
// The length of the supplied input must match the end of a block exactly.
// The *capacity* of the dst slice must match the destination size of
// the uncompressed data exactly.
func (d *Decoder) Decompress4X(dst, src []byte) ([]byte, error) {
if len(d.dt.single) == 0 {
return nil, errors.New("no table loaded")
}
if len(src) < 6+(4*1) {
return nil, errors.New("input too small")
}
if use8BitTables && d.actualTableLog <= 8 {
return d.decompress4X8bit(dst, src)
}
var br [4]bitReaderShifted
start := 6
for i := 0; i < 3; i++ {
length := int(src[i*2]) | (int(src[i*2+1]) << 8)
if start+length >= len(src) {
return nil, errors.New("truncated input (or invalid offset)")
}
err := br[i].init(src[start : start+length])
if err != nil {
return nil, err
}
start += length
}
err := br[3].init(src[start:])
if err != nil {
return nil, err
}
// destination, offset to match first output
dstSize := cap(dst)
dst = dst[:dstSize]
out := dst
dstEvery := (dstSize + 3) / 4
const tlSize = 1 << tableLogMax
const tlMask = tlSize - 1
single := d.dt.single[:tlSize]
// Use temp table to avoid bound checks/append penalty.
var buf [256]byte
var off uint8
var decoded int
// Decode 2 values from each decoder/loop.
const bufoff = 256 / 4
for {
if br[0].off < 4 || br[1].off < 4 || br[2].off < 4 || br[3].off < 4 {
break
}
{
const stream = 0
const stream2 = 1
br[stream].fillFast()
br[stream2].fillFast()
val := br[stream].peekBitsFast(d.actualTableLog)
val2 := br[stream2].peekBitsFast(d.actualTableLog)
v := single[val&tlMask]
v2 := single[val2&tlMask]
br[stream].advance(uint8(v.entry))
br[stream2].advance(uint8(v2.entry))
buf[off+bufoff*stream] = uint8(v.entry >> 8)
buf[off+bufoff*stream2] = uint8(v2.entry >> 8)
val = br[stream].peekBitsFast(d.actualTableLog)
val2 = br[stream2].peekBitsFast(d.actualTableLog)
v = single[val&tlMask]
v2 = single[val2&tlMask]
br[stream].advance(uint8(v.entry))
br[stream2].advance(uint8(v2.entry))
buf[off+bufoff*stream+1] = uint8(v.entry >> 8)
buf[off+bufoff*stream2+1] = uint8(v2.entry >> 8)
}
{
const stream = 2
const stream2 = 3
br[stream].fillFast()
br[stream2].fillFast()
val := br[stream].peekBitsFast(d.actualTableLog)
val2 := br[stream2].peekBitsFast(d.actualTableLog)
v := single[val&tlMask]
v2 := single[val2&tlMask]
br[stream].advance(uint8(v.entry))
br[stream2].advance(uint8(v2.entry))
buf[off+bufoff*stream] = uint8(v.entry >> 8)
buf[off+bufoff*stream2] = uint8(v2.entry >> 8)
val = br[stream].peekBitsFast(d.actualTableLog)
val2 = br[stream2].peekBitsFast(d.actualTableLog)
v = single[val&tlMask]
v2 = single[val2&tlMask]
br[stream].advance(uint8(v.entry))
br[stream2].advance(uint8(v2.entry))
buf[off+bufoff*stream+1] = uint8(v.entry >> 8)
buf[off+bufoff*stream2+1] = uint8(v2.entry >> 8)
}
off += 2
if off == bufoff {
if bufoff > dstEvery {
return nil, errors.New("corruption detected: stream overrun 1")
}
copy(out, buf[:bufoff])
copy(out[dstEvery:], buf[bufoff:bufoff*2])
copy(out[dstEvery*2:], buf[bufoff*2:bufoff*3])
copy(out[dstEvery*3:], buf[bufoff*3:bufoff*4])
off = 0
out = out[bufoff:]
decoded += 256
// There must at least be 3 buffers left.
if len(out) < dstEvery*3 {
return nil, errors.New("corruption detected: stream overrun 2")
}
}
}
if off > 0 {
ioff := int(off)
if len(out) < dstEvery*3+ioff {
return nil, errors.New("corruption detected: stream overrun 3")
}
copy(out, buf[:off])
copy(out[dstEvery:dstEvery+ioff], buf[bufoff:bufoff*2])
copy(out[dstEvery*2:dstEvery*2+ioff], buf[bufoff*2:bufoff*3])
copy(out[dstEvery*3:dstEvery*3+ioff], buf[bufoff*3:bufoff*4])
decoded += int(off) * 4
out = out[off:]
}
// Decode remaining.
for i := range br {
offset := dstEvery * i
br := &br[i]
bitsLeft := br.off*8 + uint(64-br.bitsRead)
for bitsLeft > 0 {
br.fill()
if false && br.bitsRead >= 32 {
if br.off >= 4 {
v := br.in[br.off-4:]
v = v[:4]
low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24)
br.value = (br.value << 32) | uint64(low)
br.bitsRead -= 32
br.off -= 4
} else {
for br.off > 0 {
br.value = (br.value << 8) | uint64(br.in[br.off-1])
br.bitsRead -= 8
br.off--
}
}
}
// end inline...
if offset >= len(out) {
return nil, errors.New("corruption detected: stream overrun 4")
}
// Read value and increment offset.
val := br.peekBitsFast(d.actualTableLog)
v := single[val&tlMask].entry
nBits := uint8(v)
br.advance(nBits)
bitsLeft -= uint(nBits)
out[offset] = uint8(v >> 8)
offset++
}
decoded += offset - dstEvery*i
err = br.close()
if err != nil {
return nil, err
}
}
if dstSize != decoded {
return nil, errors.New("corruption detected: short output block")
}
return dst, nil
}
// Decompress4X will decompress a 4X encoded stream. // Decompress4X will decompress a 4X encoded stream.
// The length of the supplied input must match the end of a block exactly. // The length of the supplied input must match the end of a block exactly.
// The *capacity* of the dst slice must match the destination size of // The *capacity* of the dst slice must match the destination size of
@ -916,12 +764,12 @@ func (d *Decoder) decompress4X8bit(dst, src []byte) ([]byte, error) {
single := d.dt.single[:tlSize] single := d.dt.single[:tlSize]
// Use temp table to avoid bound checks/append penalty. // Use temp table to avoid bound checks/append penalty.
var buf [256]byte buf := d.buffer()
var off uint8 var off uint8
var decoded int var decoded int
// Decode 4 values from each decoder/loop. // Decode 4 values from each decoder/loop.
const bufoff = 256 / 4 const bufoff = 256
for { for {
if br[0].off < 4 || br[1].off < 4 || br[2].off < 4 || br[3].off < 4 { if br[0].off < 4 || br[1].off < 4 || br[2].off < 4 || br[3].off < 4 {
break break
@ -942,8 +790,8 @@ func (d *Decoder) decompress4X8bit(dst, src []byte) ([]byte, error) {
br1.value <<= v & 63 br1.value <<= v & 63
br2.bitsRead += uint8(v2) br2.bitsRead += uint8(v2)
br2.value <<= v2 & 63 br2.value <<= v2 & 63
buf[off+bufoff*stream] = uint8(v >> 8) buf[stream][off] = uint8(v >> 8)
buf[off+bufoff*stream2] = uint8(v2 >> 8) buf[stream2][off] = uint8(v2 >> 8)
v = single[uint8(br1.value>>shift)].entry v = single[uint8(br1.value>>shift)].entry
v2 = single[uint8(br2.value>>shift)].entry v2 = single[uint8(br2.value>>shift)].entry
@ -951,8 +799,8 @@ func (d *Decoder) decompress4X8bit(dst, src []byte) ([]byte, error) {
br1.value <<= v & 63 br1.value <<= v & 63
br2.bitsRead += uint8(v2) br2.bitsRead += uint8(v2)
br2.value <<= v2 & 63 br2.value <<= v2 & 63
buf[off+bufoff*stream+1] = uint8(v >> 8) buf[stream][off+1] = uint8(v >> 8)
buf[off+bufoff*stream2+1] = uint8(v2 >> 8) buf[stream2][off+1] = uint8(v2 >> 8)
v = single[uint8(br1.value>>shift)].entry v = single[uint8(br1.value>>shift)].entry
v2 = single[uint8(br2.value>>shift)].entry v2 = single[uint8(br2.value>>shift)].entry
@ -960,8 +808,8 @@ func (d *Decoder) decompress4X8bit(dst, src []byte) ([]byte, error) {
br1.value <<= v & 63 br1.value <<= v & 63
br2.bitsRead += uint8(v2) br2.bitsRead += uint8(v2)
br2.value <<= v2 & 63 br2.value <<= v2 & 63
buf[off+bufoff*stream+2] = uint8(v >> 8) buf[stream][off+2] = uint8(v >> 8)
buf[off+bufoff*stream2+2] = uint8(v2 >> 8) buf[stream2][off+2] = uint8(v2 >> 8)
v = single[uint8(br1.value>>shift)].entry v = single[uint8(br1.value>>shift)].entry
v2 = single[uint8(br2.value>>shift)].entry v2 = single[uint8(br2.value>>shift)].entry
@ -969,8 +817,8 @@ func (d *Decoder) decompress4X8bit(dst, src []byte) ([]byte, error) {
br1.value <<= v & 63 br1.value <<= v & 63
br2.bitsRead += uint8(v2) br2.bitsRead += uint8(v2)
br2.value <<= v2 & 63 br2.value <<= v2 & 63
buf[off+bufoff*stream2+3] = uint8(v2 >> 8) buf[stream][off+3] = uint8(v >> 8)
buf[off+bufoff*stream+3] = uint8(v >> 8) buf[stream2][off+3] = uint8(v2 >> 8)
} }
{ {
@ -987,8 +835,8 @@ func (d *Decoder) decompress4X8bit(dst, src []byte) ([]byte, error) {
br1.value <<= v & 63 br1.value <<= v & 63
br2.bitsRead += uint8(v2) br2.bitsRead += uint8(v2)
br2.value <<= v2 & 63 br2.value <<= v2 & 63
buf[off+bufoff*stream] = uint8(v >> 8) buf[stream][off] = uint8(v >> 8)
buf[off+bufoff*stream2] = uint8(v2 >> 8) buf[stream2][off] = uint8(v2 >> 8)
v = single[uint8(br1.value>>shift)].entry v = single[uint8(br1.value>>shift)].entry
v2 = single[uint8(br2.value>>shift)].entry v2 = single[uint8(br2.value>>shift)].entry
@ -996,8 +844,8 @@ func (d *Decoder) decompress4X8bit(dst, src []byte) ([]byte, error) {
br1.value <<= v & 63 br1.value <<= v & 63
br2.bitsRead += uint8(v2) br2.bitsRead += uint8(v2)
br2.value <<= v2 & 63 br2.value <<= v2 & 63
buf[off+bufoff*stream+1] = uint8(v >> 8) buf[stream][off+1] = uint8(v >> 8)
buf[off+bufoff*stream2+1] = uint8(v2 >> 8) buf[stream2][off+1] = uint8(v2 >> 8)
v = single[uint8(br1.value>>shift)].entry v = single[uint8(br1.value>>shift)].entry
v2 = single[uint8(br2.value>>shift)].entry v2 = single[uint8(br2.value>>shift)].entry
@ -1005,8 +853,8 @@ func (d *Decoder) decompress4X8bit(dst, src []byte) ([]byte, error) {
br1.value <<= v & 63 br1.value <<= v & 63
br2.bitsRead += uint8(v2) br2.bitsRead += uint8(v2)
br2.value <<= v2 & 63 br2.value <<= v2 & 63
buf[off+bufoff*stream+2] = uint8(v >> 8) buf[stream][off+2] = uint8(v >> 8)
buf[off+bufoff*stream2+2] = uint8(v2 >> 8) buf[stream2][off+2] = uint8(v2 >> 8)
v = single[uint8(br1.value>>shift)].entry v = single[uint8(br1.value>>shift)].entry
v2 = single[uint8(br2.value>>shift)].entry v2 = single[uint8(br2.value>>shift)].entry
@ -1014,25 +862,26 @@ func (d *Decoder) decompress4X8bit(dst, src []byte) ([]byte, error) {
br1.value <<= v & 63 br1.value <<= v & 63
br2.bitsRead += uint8(v2) br2.bitsRead += uint8(v2)
br2.value <<= v2 & 63 br2.value <<= v2 & 63
buf[off+bufoff*stream2+3] = uint8(v2 >> 8) buf[stream][off+3] = uint8(v >> 8)
buf[off+bufoff*stream+3] = uint8(v >> 8) buf[stream2][off+3] = uint8(v2 >> 8)
} }
off += 4 off += 4
if off == bufoff { if off == 0 {
if bufoff > dstEvery { if bufoff > dstEvery {
d.bufs.Put(buf)
return nil, errors.New("corruption detected: stream overrun 1") return nil, errors.New("corruption detected: stream overrun 1")
} }
copy(out, buf[:bufoff]) copy(out, buf[0][:])
copy(out[dstEvery:], buf[bufoff:bufoff*2]) copy(out[dstEvery:], buf[1][:])
copy(out[dstEvery*2:], buf[bufoff*2:bufoff*3]) copy(out[dstEvery*2:], buf[2][:])
copy(out[dstEvery*3:], buf[bufoff*3:bufoff*4]) copy(out[dstEvery*3:], buf[3][:])
off = 0
out = out[bufoff:] out = out[bufoff:]
decoded += 256 decoded += bufoff * 4
// There must at least be 3 buffers left. // There must at least be 3 buffers left.
if len(out) < dstEvery*3 { if len(out) < dstEvery*3 {
d.bufs.Put(buf)
return nil, errors.New("corruption detected: stream overrun 2") return nil, errors.New("corruption detected: stream overrun 2")
} }
} }
@ -1040,23 +889,31 @@ func (d *Decoder) decompress4X8bit(dst, src []byte) ([]byte, error) {
if off > 0 { if off > 0 {
ioff := int(off) ioff := int(off)
if len(out) < dstEvery*3+ioff { if len(out) < dstEvery*3+ioff {
d.bufs.Put(buf)
return nil, errors.New("corruption detected: stream overrun 3") return nil, errors.New("corruption detected: stream overrun 3")
} }
copy(out, buf[:off]) copy(out, buf[0][:off])
copy(out[dstEvery:dstEvery+ioff], buf[bufoff:bufoff*2]) copy(out[dstEvery:], buf[1][:off])
copy(out[dstEvery*2:dstEvery*2+ioff], buf[bufoff*2:bufoff*3]) copy(out[dstEvery*2:], buf[2][:off])
copy(out[dstEvery*3:dstEvery*3+ioff], buf[bufoff*3:bufoff*4]) copy(out[dstEvery*3:], buf[3][:off])
decoded += int(off) * 4 decoded += int(off) * 4
out = out[off:] out = out[off:]
} }
// Decode remaining. // Decode remaining.
// Decode remaining.
remainBytes := dstEvery - (decoded / 4)
for i := range br { for i := range br {
offset := dstEvery * i offset := dstEvery * i
endsAt := offset + remainBytes
if endsAt > len(out) {
endsAt = len(out)
}
br := &br[i] br := &br[i]
bitsLeft := int(br.off*8) + int(64-br.bitsRead) bitsLeft := br.remaining()
for bitsLeft > 0 { for bitsLeft > 0 {
if br.finished() { if br.finished() {
d.bufs.Put(buf)
return nil, io.ErrUnexpectedEOF return nil, io.ErrUnexpectedEOF
} }
if br.bitsRead >= 56 { if br.bitsRead >= 56 {
@ -1076,7 +933,8 @@ func (d *Decoder) decompress4X8bit(dst, src []byte) ([]byte, error) {
} }
} }
// end inline... // end inline...
if offset >= len(out) { if offset >= endsAt {
d.bufs.Put(buf)
return nil, errors.New("corruption detected: stream overrun 4") return nil, errors.New("corruption detected: stream overrun 4")
} }
@ -1084,16 +942,22 @@ func (d *Decoder) decompress4X8bit(dst, src []byte) ([]byte, error) {
v := single[uint8(br.value>>shift)].entry v := single[uint8(br.value>>shift)].entry
nBits := uint8(v) nBits := uint8(v)
br.advance(nBits) br.advance(nBits)
bitsLeft -= int(nBits) bitsLeft -= uint(nBits)
out[offset] = uint8(v >> 8) out[offset] = uint8(v >> 8)
offset++ offset++
} }
if offset != endsAt {
d.bufs.Put(buf)
return nil, fmt.Errorf("corruption detected: short output block %d, end %d != %d", i, offset, endsAt)
}
decoded += offset - dstEvery*i decoded += offset - dstEvery*i
err = br.close() err = br.close()
if err != nil { if err != nil {
d.bufs.Put(buf)
return nil, err return nil, err
} }
} }
d.bufs.Put(buf)
if dstSize != decoded { if dstSize != decoded {
return nil, errors.New("corruption detected: short output block") return nil, errors.New("corruption detected: short output block")
} }
@ -1135,12 +999,12 @@ func (d *Decoder) decompress4X8bitExactly(dst, src []byte) ([]byte, error) {
single := d.dt.single[:tlSize] single := d.dt.single[:tlSize]
// Use temp table to avoid bound checks/append penalty. // Use temp table to avoid bound checks/append penalty.
var buf [256]byte buf := d.buffer()
var off uint8 var off uint8
var decoded int var decoded int
// Decode 4 values from each decoder/loop. // Decode 4 values from each decoder/loop.
const bufoff = 256 / 4 const bufoff = 256
for { for {
if br[0].off < 4 || br[1].off < 4 || br[2].off < 4 || br[3].off < 4 { if br[0].off < 4 || br[1].off < 4 || br[2].off < 4 || br[3].off < 4 {
break break
@ -1150,104 +1014,109 @@ func (d *Decoder) decompress4X8bitExactly(dst, src []byte) ([]byte, error) {
// Interleave 2 decodes. // Interleave 2 decodes.
const stream = 0 const stream = 0
const stream2 = 1 const stream2 = 1
br[stream].fillFast() br1 := &br[stream]
br[stream2].fillFast() br2 := &br[stream2]
br1.fillFast()
br2.fillFast()
v := single[uint8(br[stream].value>>shift)].entry v := single[uint8(br1.value>>shift)].entry
v2 := single[uint8(br[stream2].value>>shift)].entry v2 := single[uint8(br2.value>>shift)].entry
br[stream].bitsRead += uint8(v) br1.bitsRead += uint8(v)
br[stream].value <<= v & 63 br1.value <<= v & 63
br[stream2].bitsRead += uint8(v2) br2.bitsRead += uint8(v2)
br[stream2].value <<= v2 & 63 br2.value <<= v2 & 63
buf[off+bufoff*stream] = uint8(v >> 8) buf[stream][off] = uint8(v >> 8)
buf[off+bufoff*stream2] = uint8(v2 >> 8) buf[stream2][off] = uint8(v2 >> 8)
v = single[uint8(br[stream].value>>shift)].entry v = single[uint8(br1.value>>shift)].entry
v2 = single[uint8(br[stream2].value>>shift)].entry v2 = single[uint8(br2.value>>shift)].entry
br[stream].bitsRead += uint8(v) br1.bitsRead += uint8(v)
br[stream].value <<= v & 63 br1.value <<= v & 63
br[stream2].bitsRead += uint8(v2) br2.bitsRead += uint8(v2)
br[stream2].value <<= v2 & 63 br2.value <<= v2 & 63
buf[off+bufoff*stream+1] = uint8(v >> 8) buf[stream][off+1] = uint8(v >> 8)
buf[off+bufoff*stream2+1] = uint8(v2 >> 8) buf[stream2][off+1] = uint8(v2 >> 8)
v = single[uint8(br[stream].value>>shift)].entry v = single[uint8(br1.value>>shift)].entry
v2 = single[uint8(br[stream2].value>>shift)].entry v2 = single[uint8(br2.value>>shift)].entry
br[stream].bitsRead += uint8(v) br1.bitsRead += uint8(v)
br[stream].value <<= v & 63 br1.value <<= v & 63
br[stream2].bitsRead += uint8(v2) br2.bitsRead += uint8(v2)
br[stream2].value <<= v2 & 63 br2.value <<= v2 & 63
buf[off+bufoff*stream+2] = uint8(v >> 8) buf[stream][off+2] = uint8(v >> 8)
buf[off+bufoff*stream2+2] = uint8(v2 >> 8) buf[stream2][off+2] = uint8(v2 >> 8)
v = single[uint8(br[stream].value>>shift)].entry v = single[uint8(br1.value>>shift)].entry
v2 = single[uint8(br[stream2].value>>shift)].entry v2 = single[uint8(br2.value>>shift)].entry
br[stream].bitsRead += uint8(v) br1.bitsRead += uint8(v)
br[stream].value <<= v & 63 br1.value <<= v & 63
br[stream2].bitsRead += uint8(v2) br2.bitsRead += uint8(v2)
br[stream2].value <<= v2 & 63 br2.value <<= v2 & 63
buf[off+bufoff*stream+3] = uint8(v >> 8) buf[stream][off+3] = uint8(v >> 8)
buf[off+bufoff*stream2+3] = uint8(v2 >> 8) buf[stream2][off+3] = uint8(v2 >> 8)
} }
{ {
const stream = 2 const stream = 2
const stream2 = 3 const stream2 = 3
br[stream].fillFast() br1 := &br[stream]
br[stream2].fillFast() br2 := &br[stream2]
br1.fillFast()
br2.fillFast()
v := single[uint8(br[stream].value>>shift)].entry v := single[uint8(br1.value>>shift)].entry
v2 := single[uint8(br[stream2].value>>shift)].entry v2 := single[uint8(br2.value>>shift)].entry
br[stream].bitsRead += uint8(v) br1.bitsRead += uint8(v)
br[stream].value <<= v & 63 br1.value <<= v & 63
br[stream2].bitsRead += uint8(v2) br2.bitsRead += uint8(v2)
br[stream2].value <<= v2 & 63 br2.value <<= v2 & 63
buf[off+bufoff*stream] = uint8(v >> 8) buf[stream][off] = uint8(v >> 8)
buf[off+bufoff*stream2] = uint8(v2 >> 8) buf[stream2][off] = uint8(v2 >> 8)
v = single[uint8(br[stream].value>>shift)].entry v = single[uint8(br1.value>>shift)].entry
v2 = single[uint8(br[stream2].value>>shift)].entry v2 = single[uint8(br2.value>>shift)].entry
br[stream].bitsRead += uint8(v) br1.bitsRead += uint8(v)
br[stream].value <<= v & 63 br1.value <<= v & 63
br[stream2].bitsRead += uint8(v2) br2.bitsRead += uint8(v2)
br[stream2].value <<= v2 & 63 br2.value <<= v2 & 63
buf[off+bufoff*stream+1] = uint8(v >> 8) buf[stream][off+1] = uint8(v >> 8)
buf[off+bufoff*stream2+1] = uint8(v2 >> 8) buf[stream2][off+1] = uint8(v2 >> 8)
v = single[uint8(br[stream].value>>shift)].entry v = single[uint8(br1.value>>shift)].entry
v2 = single[uint8(br[stream2].value>>shift)].entry v2 = single[uint8(br2.value>>shift)].entry
br[stream].bitsRead += uint8(v) br1.bitsRead += uint8(v)
br[stream].value <<= v & 63 br1.value <<= v & 63
br[stream2].bitsRead += uint8(v2) br2.bitsRead += uint8(v2)
br[stream2].value <<= v2 & 63 br2.value <<= v2 & 63
buf[off+bufoff*stream+2] = uint8(v >> 8) buf[stream][off+2] = uint8(v >> 8)
buf[off+bufoff*stream2+2] = uint8(v2 >> 8) buf[stream2][off+2] = uint8(v2 >> 8)
v = single[uint8(br[stream].value>>shift)].entry v = single[uint8(br1.value>>shift)].entry
v2 = single[uint8(br[stream2].value>>shift)].entry v2 = single[uint8(br2.value>>shift)].entry
br[stream].bitsRead += uint8(v) br1.bitsRead += uint8(v)
br[stream].value <<= v & 63 br1.value <<= v & 63
br[stream2].bitsRead += uint8(v2) br2.bitsRead += uint8(v2)
br[stream2].value <<= v2 & 63 br2.value <<= v2 & 63
buf[off+bufoff*stream+3] = uint8(v >> 8) buf[stream][off+3] = uint8(v >> 8)
buf[off+bufoff*stream2+3] = uint8(v2 >> 8) buf[stream2][off+3] = uint8(v2 >> 8)
} }
off += 4 off += 4
if off == bufoff { if off == 0 {
if bufoff > dstEvery { if bufoff > dstEvery {
d.bufs.Put(buf)
return nil, errors.New("corruption detected: stream overrun 1") return nil, errors.New("corruption detected: stream overrun 1")
} }
copy(out, buf[:bufoff]) copy(out, buf[0][:])
copy(out[dstEvery:], buf[bufoff:bufoff*2]) copy(out[dstEvery:], buf[1][:])
copy(out[dstEvery*2:], buf[bufoff*2:bufoff*3]) copy(out[dstEvery*2:], buf[2][:])
copy(out[dstEvery*3:], buf[bufoff*3:bufoff*4]) copy(out[dstEvery*3:], buf[3][:])
off = 0
out = out[bufoff:] out = out[bufoff:]
decoded += 256 decoded += bufoff * 4
// There must at least be 3 buffers left. // There must at least be 3 buffers left.
if len(out) < dstEvery*3 { if len(out) < dstEvery*3 {
d.bufs.Put(buf)
return nil, errors.New("corruption detected: stream overrun 2") return nil, errors.New("corruption detected: stream overrun 2")
} }
} }
@ -1257,21 +1126,27 @@ func (d *Decoder) decompress4X8bitExactly(dst, src []byte) ([]byte, error) {
if len(out) < dstEvery*3+ioff { if len(out) < dstEvery*3+ioff {
return nil, errors.New("corruption detected: stream overrun 3") return nil, errors.New("corruption detected: stream overrun 3")
} }
copy(out, buf[:off]) copy(out, buf[0][:off])
copy(out[dstEvery:dstEvery+ioff], buf[bufoff:bufoff*2]) copy(out[dstEvery:], buf[1][:off])
copy(out[dstEvery*2:dstEvery*2+ioff], buf[bufoff*2:bufoff*3]) copy(out[dstEvery*2:], buf[2][:off])
copy(out[dstEvery*3:dstEvery*3+ioff], buf[bufoff*3:bufoff*4]) copy(out[dstEvery*3:], buf[3][:off])
decoded += int(off) * 4 decoded += int(off) * 4
out = out[off:] out = out[off:]
} }
// Decode remaining. // Decode remaining.
remainBytes := dstEvery - (decoded / 4)
for i := range br { for i := range br {
offset := dstEvery * i offset := dstEvery * i
endsAt := offset + remainBytes
if endsAt > len(out) {
endsAt = len(out)
}
br := &br[i] br := &br[i]
bitsLeft := int(br.off*8) + int(64-br.bitsRead) bitsLeft := br.remaining()
for bitsLeft > 0 { for bitsLeft > 0 {
if br.finished() { if br.finished() {
d.bufs.Put(buf)
return nil, io.ErrUnexpectedEOF return nil, io.ErrUnexpectedEOF
} }
if br.bitsRead >= 56 { if br.bitsRead >= 56 {
@ -1291,7 +1166,8 @@ func (d *Decoder) decompress4X8bitExactly(dst, src []byte) ([]byte, error) {
} }
} }
// end inline... // end inline...
if offset >= len(out) { if offset >= endsAt {
d.bufs.Put(buf)
return nil, errors.New("corruption detected: stream overrun 4") return nil, errors.New("corruption detected: stream overrun 4")
} }
@ -1299,16 +1175,23 @@ func (d *Decoder) decompress4X8bitExactly(dst, src []byte) ([]byte, error) {
v := single[br.peekByteFast()].entry v := single[br.peekByteFast()].entry
nBits := uint8(v) nBits := uint8(v)
br.advance(nBits) br.advance(nBits)
bitsLeft -= int(nBits) bitsLeft -= uint(nBits)
out[offset] = uint8(v >> 8) out[offset] = uint8(v >> 8)
offset++ offset++
} }
if offset != endsAt {
d.bufs.Put(buf)
return nil, fmt.Errorf("corruption detected: short output block %d, end %d != %d", i, offset, endsAt)
}
decoded += offset - dstEvery*i decoded += offset - dstEvery*i
err = br.close() err = br.close()
if err != nil { if err != nil {
d.bufs.Put(buf)
return nil, err return nil, err
} }
} }
d.bufs.Put(buf)
if dstSize != decoded { if dstSize != decoded {
return nil, errors.New("corruption detected: short output block") return nil, errors.New("corruption detected: short output block")
} }

488
vendor/github.com/klauspost/compress/huff0/decompress_8b_amd64.s generated vendored Normal file
View File

@ -0,0 +1,488 @@
// +build !appengine
// +build gc
// +build !noasm
#include "textflag.h"
#include "funcdata.h"
#include "go_asm.h"
#define bufoff 256 // see decompress.go, we're using [4][256]byte table
// func decompress4x_main_loop_x86(pbr0, pbr1, pbr2, pbr3 *bitReaderShifted,
// peekBits uint8, buf *byte, tbl *dEntrySingle) (int, bool)
TEXT ·decompress4x_8b_loop_x86(SB), NOSPLIT, $8
#define off R8
#define buffer DI
#define table SI
#define br_bits_read R9
#define br_value R10
#define br_offset R11
#define peek_bits R12
#define exhausted DX
#define br0 R13
#define br1 R14
#define br2 R15
#define br3 BP
MOVQ BP, 0(SP)
XORQ exhausted, exhausted // exhausted = false
XORQ off, off // off = 0
MOVBQZX peekBits+32(FP), peek_bits
MOVQ buf+40(FP), buffer
MOVQ tbl+48(FP), table
MOVQ pbr0+0(FP), br0
MOVQ pbr1+8(FP), br1
MOVQ pbr2+16(FP), br2
MOVQ pbr3+24(FP), br3
main_loop:
// const stream = 0
// br0.fillFast()
MOVBQZX bitReaderShifted_bitsRead(br0), br_bits_read
MOVQ bitReaderShifted_value(br0), br_value
MOVQ bitReaderShifted_off(br0), br_offset
// if b.bitsRead >= 32 {
CMPQ br_bits_read, $32
JB skip_fill0
SUBQ $32, br_bits_read // b.bitsRead -= 32
SUBQ $4, br_offset // b.off -= 4
// v := b.in[b.off-4 : b.off]
// v = v[:4]
// low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24)
MOVQ bitReaderShifted_in(br0), AX
MOVL 0(br_offset)(AX*1), AX // AX = uint32(b.in[b.off:b.off+4])
// b.value |= uint64(low) << (b.bitsRead & 63)
MOVQ br_bits_read, CX
SHLQ CL, AX
ORQ AX, br_value
// exhausted = exhausted || (br0.off < 4)
CMPQ br_offset, $4
SETLT DL
ORB DL, DH
// }
skip_fill0:
// val0 := br0.peekTopBits(peekBits)
MOVQ br_value, AX
MOVQ peek_bits, CX
SHRQ CL, AX // AX = (value >> peek_bits) & mask
// v0 := table[val0&mask]
MOVW 0(table)(AX*2), AX // AX - v0
// br0.advance(uint8(v0.entry))
MOVB AH, BL // BL = uint8(v0.entry >> 8)
MOVBQZX AL, CX
SHLQ CL, br_value // value <<= n
ADDQ CX, br_bits_read // bits_read += n
// val1 := br0.peekTopBits(peekBits)
MOVQ peek_bits, CX
MOVQ br_value, AX
SHRQ CL, AX // AX = (value >> peek_bits) & mask
// v1 := table[val1&mask]
MOVW 0(table)(AX*2), AX // AX - v1
// br0.advance(uint8(v1.entry))
MOVB AH, BH // BH = uint8(v1.entry >> 8)
MOVBQZX AL, CX
SHLQ CX, br_value // value <<= n
ADDQ CX, br_bits_read // bits_read += n
// these two writes get coalesced
// buf[stream][off] = uint8(v0.entry >> 8)
// buf[stream][off+1] = uint8(v1.entry >> 8)
MOVW BX, 0(buffer)(off*1)
// SECOND PART:
// val2 := br0.peekTopBits(peekBits)
MOVQ br_value, AX
MOVQ peek_bits, CX
SHRQ CL, AX // AX = (value >> peek_bits) & mask
// v2 := table[val0&mask]
MOVW 0(table)(AX*2), AX // AX - v0
// br0.advance(uint8(v0.entry))
MOVB AH, BL // BL = uint8(v0.entry >> 8)
MOVBQZX AL, CX
SHLQ CL, br_value // value <<= n
ADDQ CX, br_bits_read // bits_read += n
// val3 := br0.peekTopBits(peekBits)
MOVQ peek_bits, CX
MOVQ br_value, AX
SHRQ CL, AX // AX = (value >> peek_bits) & mask
// v3 := table[val1&mask]
MOVW 0(table)(AX*2), AX // AX - v1
// br0.advance(uint8(v1.entry))
MOVB AH, BH // BH = uint8(v1.entry >> 8)
MOVBQZX AL, CX
SHLQ CX, br_value // value <<= n
ADDQ CX, br_bits_read // bits_read += n
// these two writes get coalesced
// buf[stream][off+2] = uint8(v2.entry >> 8)
// buf[stream][off+3] = uint8(v3.entry >> 8)
MOVW BX, 0+2(buffer)(off*1)
// update the bitrader reader structure
MOVB br_bits_read, bitReaderShifted_bitsRead(br0)
MOVQ br_value, bitReaderShifted_value(br0)
MOVQ br_offset, bitReaderShifted_off(br0)
// const stream = 1
// br1.fillFast()
MOVBQZX bitReaderShifted_bitsRead(br1), br_bits_read
MOVQ bitReaderShifted_value(br1), br_value
MOVQ bitReaderShifted_off(br1), br_offset
// if b.bitsRead >= 32 {
CMPQ br_bits_read, $32
JB skip_fill1
SUBQ $32, br_bits_read // b.bitsRead -= 32
SUBQ $4, br_offset // b.off -= 4
// v := b.in[b.off-4 : b.off]
// v = v[:4]
// low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24)
MOVQ bitReaderShifted_in(br1), AX
MOVL 0(br_offset)(AX*1), AX // AX = uint32(b.in[b.off:b.off+4])
// b.value |= uint64(low) << (b.bitsRead & 63)
MOVQ br_bits_read, CX
SHLQ CL, AX
ORQ AX, br_value
// exhausted = exhausted || (br1.off < 4)
CMPQ br_offset, $4
SETLT DL
ORB DL, DH
// }
skip_fill1:
// val0 := br1.peekTopBits(peekBits)
MOVQ br_value, AX
MOVQ peek_bits, CX
SHRQ CL, AX // AX = (value >> peek_bits) & mask
// v0 := table[val0&mask]
MOVW 0(table)(AX*2), AX // AX - v0
// br1.advance(uint8(v0.entry))
MOVB AH, BL // BL = uint8(v0.entry >> 8)
MOVBQZX AL, CX
SHLQ CL, br_value // value <<= n
ADDQ CX, br_bits_read // bits_read += n
// val1 := br1.peekTopBits(peekBits)
MOVQ peek_bits, CX
MOVQ br_value, AX
SHRQ CL, AX // AX = (value >> peek_bits) & mask
// v1 := table[val1&mask]
MOVW 0(table)(AX*2), AX // AX - v1
// br1.advance(uint8(v1.entry))
MOVB AH, BH // BH = uint8(v1.entry >> 8)
MOVBQZX AL, CX
SHLQ CX, br_value // value <<= n
ADDQ CX, br_bits_read // bits_read += n
// these two writes get coalesced
// buf[stream][off] = uint8(v0.entry >> 8)
// buf[stream][off+1] = uint8(v1.entry >> 8)
MOVW BX, 256(buffer)(off*1)
// SECOND PART:
// val2 := br1.peekTopBits(peekBits)
MOVQ br_value, AX
MOVQ peek_bits, CX
SHRQ CL, AX // AX = (value >> peek_bits) & mask
// v2 := table[val0&mask]
MOVW 0(table)(AX*2), AX // AX - v0
// br1.advance(uint8(v0.entry))
MOVB AH, BL // BL = uint8(v0.entry >> 8)
MOVBQZX AL, CX
SHLQ CL, br_value // value <<= n
ADDQ CX, br_bits_read // bits_read += n
// val3 := br1.peekTopBits(peekBits)
MOVQ peek_bits, CX
MOVQ br_value, AX
SHRQ CL, AX // AX = (value >> peek_bits) & mask
// v3 := table[val1&mask]
MOVW 0(table)(AX*2), AX // AX - v1
// br1.advance(uint8(v1.entry))
MOVB AH, BH // BH = uint8(v1.entry >> 8)
MOVBQZX AL, CX
SHLQ CX, br_value // value <<= n
ADDQ CX, br_bits_read // bits_read += n
// these two writes get coalesced
// buf[stream][off+2] = uint8(v2.entry >> 8)
// buf[stream][off+3] = uint8(v3.entry >> 8)
MOVW BX, 256+2(buffer)(off*1)
// update the bitrader reader structure
MOVB br_bits_read, bitReaderShifted_bitsRead(br1)
MOVQ br_value, bitReaderShifted_value(br1)
MOVQ br_offset, bitReaderShifted_off(br1)
// const stream = 2
// br2.fillFast()
MOVBQZX bitReaderShifted_bitsRead(br2), br_bits_read
MOVQ bitReaderShifted_value(br2), br_value
MOVQ bitReaderShifted_off(br2), br_offset
// if b.bitsRead >= 32 {
CMPQ br_bits_read, $32
JB skip_fill2
SUBQ $32, br_bits_read // b.bitsRead -= 32
SUBQ $4, br_offset // b.off -= 4
// v := b.in[b.off-4 : b.off]
// v = v[:4]
// low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24)
MOVQ bitReaderShifted_in(br2), AX
MOVL 0(br_offset)(AX*1), AX // AX = uint32(b.in[b.off:b.off+4])
// b.value |= uint64(low) << (b.bitsRead & 63)
MOVQ br_bits_read, CX
SHLQ CL, AX
ORQ AX, br_value
// exhausted = exhausted || (br2.off < 4)
CMPQ br_offset, $4
SETLT DL
ORB DL, DH
// }
skip_fill2:
// val0 := br2.peekTopBits(peekBits)
MOVQ br_value, AX
MOVQ peek_bits, CX
SHRQ CL, AX // AX = (value >> peek_bits) & mask
// v0 := table[val0&mask]
MOVW 0(table)(AX*2), AX // AX - v0
// br2.advance(uint8(v0.entry))
MOVB AH, BL // BL = uint8(v0.entry >> 8)
MOVBQZX AL, CX
SHLQ CL, br_value // value <<= n
ADDQ CX, br_bits_read // bits_read += n
// val1 := br2.peekTopBits(peekBits)
MOVQ peek_bits, CX
MOVQ br_value, AX
SHRQ CL, AX // AX = (value >> peek_bits) & mask
// v1 := table[val1&mask]
MOVW 0(table)(AX*2), AX // AX - v1
// br2.advance(uint8(v1.entry))
MOVB AH, BH // BH = uint8(v1.entry >> 8)
MOVBQZX AL, CX
SHLQ CX, br_value // value <<= n
ADDQ CX, br_bits_read // bits_read += n
// these two writes get coalesced
// buf[stream][off] = uint8(v0.entry >> 8)
// buf[stream][off+1] = uint8(v1.entry >> 8)
MOVW BX, 512(buffer)(off*1)
// SECOND PART:
// val2 := br2.peekTopBits(peekBits)
MOVQ br_value, AX
MOVQ peek_bits, CX
SHRQ CL, AX // AX = (value >> peek_bits) & mask
// v2 := table[val0&mask]
MOVW 0(table)(AX*2), AX // AX - v0
// br2.advance(uint8(v0.entry))
MOVB AH, BL // BL = uint8(v0.entry >> 8)
MOVBQZX AL, CX
SHLQ CL, br_value // value <<= n
ADDQ CX, br_bits_read // bits_read += n
// val3 := br2.peekTopBits(peekBits)
MOVQ peek_bits, CX
MOVQ br_value, AX
SHRQ CL, AX // AX = (value >> peek_bits) & mask
// v3 := table[val1&mask]
MOVW 0(table)(AX*2), AX // AX - v1
// br2.advance(uint8(v1.entry))
MOVB AH, BH // BH = uint8(v1.entry >> 8)
MOVBQZX AL, CX
SHLQ CX, br_value // value <<= n
ADDQ CX, br_bits_read // bits_read += n
// these two writes get coalesced
// buf[stream][off+2] = uint8(v2.entry >> 8)
// buf[stream][off+3] = uint8(v3.entry >> 8)
MOVW BX, 512+2(buffer)(off*1)
// update the bitrader reader structure
MOVB br_bits_read, bitReaderShifted_bitsRead(br2)
MOVQ br_value, bitReaderShifted_value(br2)
MOVQ br_offset, bitReaderShifted_off(br2)
// const stream = 3
// br3.fillFast()
MOVBQZX bitReaderShifted_bitsRead(br3), br_bits_read
MOVQ bitReaderShifted_value(br3), br_value
MOVQ bitReaderShifted_off(br3), br_offset
// if b.bitsRead >= 32 {
CMPQ br_bits_read, $32
JB skip_fill3
SUBQ $32, br_bits_read // b.bitsRead -= 32
SUBQ $4, br_offset // b.off -= 4
// v := b.in[b.off-4 : b.off]
// v = v[:4]
// low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24)
MOVQ bitReaderShifted_in(br3), AX
MOVL 0(br_offset)(AX*1), AX // AX = uint32(b.in[b.off:b.off+4])
// b.value |= uint64(low) << (b.bitsRead & 63)
MOVQ br_bits_read, CX
SHLQ CL, AX
ORQ AX, br_value
// exhausted = exhausted || (br3.off < 4)
CMPQ br_offset, $4
SETLT DL
ORB DL, DH
// }
skip_fill3:
// val0 := br3.peekTopBits(peekBits)
MOVQ br_value, AX
MOVQ peek_bits, CX
SHRQ CL, AX // AX = (value >> peek_bits) & mask
// v0 := table[val0&mask]
MOVW 0(table)(AX*2), AX // AX - v0
// br3.advance(uint8(v0.entry))
MOVB AH, BL // BL = uint8(v0.entry >> 8)
MOVBQZX AL, CX
SHLQ CL, br_value // value <<= n
ADDQ CX, br_bits_read // bits_read += n
// val1 := br3.peekTopBits(peekBits)
MOVQ peek_bits, CX
MOVQ br_value, AX
SHRQ CL, AX // AX = (value >> peek_bits) & mask
// v1 := table[val1&mask]
MOVW 0(table)(AX*2), AX // AX - v1
// br3.advance(uint8(v1.entry))
MOVB AH, BH // BH = uint8(v1.entry >> 8)
MOVBQZX AL, CX
SHLQ CX, br_value // value <<= n
ADDQ CX, br_bits_read // bits_read += n
// these two writes get coalesced
// buf[stream][off] = uint8(v0.entry >> 8)
// buf[stream][off+1] = uint8(v1.entry >> 8)
MOVW BX, 768(buffer)(off*1)
// SECOND PART:
// val2 := br3.peekTopBits(peekBits)
MOVQ br_value, AX
MOVQ peek_bits, CX
SHRQ CL, AX // AX = (value >> peek_bits) & mask
// v2 := table[val0&mask]
MOVW 0(table)(AX*2), AX // AX - v0
// br3.advance(uint8(v0.entry))
MOVB AH, BL // BL = uint8(v0.entry >> 8)
MOVBQZX AL, CX
SHLQ CL, br_value // value <<= n
ADDQ CX, br_bits_read // bits_read += n
// val3 := br3.peekTopBits(peekBits)
MOVQ peek_bits, CX
MOVQ br_value, AX
SHRQ CL, AX // AX = (value >> peek_bits) & mask
// v3 := table[val1&mask]
MOVW 0(table)(AX*2), AX // AX - v1
// br3.advance(uint8(v1.entry))
MOVB AH, BH // BH = uint8(v1.entry >> 8)
MOVBQZX AL, CX
SHLQ CX, br_value // value <<= n
ADDQ CX, br_bits_read // bits_read += n
// these two writes get coalesced
// buf[stream][off+2] = uint8(v2.entry >> 8)
// buf[stream][off+3] = uint8(v3.entry >> 8)
MOVW BX, 768+2(buffer)(off*1)
// update the bitrader reader structure
MOVB br_bits_read, bitReaderShifted_bitsRead(br3)
MOVQ br_value, bitReaderShifted_value(br3)
MOVQ br_offset, bitReaderShifted_off(br3)
ADDQ $4, off // off += 2
TESTB DH, DH // any br[i].ofs < 4?
JNZ end
CMPQ off, $bufoff
JL main_loop
end:
MOVQ 0(SP), BP
MOVB off, ret+56(FP)
RET
#undef off
#undef buffer
#undef table
#undef br_bits_read
#undef br_value
#undef br_offset
#undef peek_bits
#undef exhausted
#undef br0
#undef br1
#undef br2
#undef br3

197
vendor/github.com/klauspost/compress/huff0/decompress_8b_amd64.s.in generated vendored Normal file
View File

@ -0,0 +1,197 @@
// +build !appengine
// +build gc
// +build !noasm
#include "textflag.h"
#include "funcdata.h"
#include "go_asm.h"
#define bufoff 256 // see decompress.go, we're using [4][256]byte table
//func decompress4x_main_loop_x86(pbr0, pbr1, pbr2, pbr3 *bitReaderShifted,
// peekBits uint8, buf *byte, tbl *dEntrySingle) (int, bool)
TEXT ·decompress4x_8b_loop_x86(SB), NOSPLIT, $8
#define off R8
#define buffer DI
#define table SI
#define br_bits_read R9
#define br_value R10
#define br_offset R11
#define peek_bits R12
#define exhausted DX
#define br0 R13
#define br1 R14
#define br2 R15
#define br3 BP
MOVQ BP, 0(SP)
XORQ exhausted, exhausted // exhausted = false
XORQ off, off // off = 0
MOVBQZX peekBits+32(FP), peek_bits
MOVQ buf+40(FP), buffer
MOVQ tbl+48(FP), table
MOVQ pbr0+0(FP), br0
MOVQ pbr1+8(FP), br1
MOVQ pbr2+16(FP), br2
MOVQ pbr3+24(FP), br3
main_loop:
{{ define "decode_2_values_x86" }}
// const stream = {{ var "id" }}
// br{{ var "id"}}.fillFast()
MOVBQZX bitReaderShifted_bitsRead(br{{ var "id" }}), br_bits_read
MOVQ bitReaderShifted_value(br{{ var "id" }}), br_value
MOVQ bitReaderShifted_off(br{{ var "id" }}), br_offset
// if b.bitsRead >= 32 {
CMPQ br_bits_read, $32
JB skip_fill{{ var "id" }}
SUBQ $32, br_bits_read // b.bitsRead -= 32
SUBQ $4, br_offset // b.off -= 4
// v := b.in[b.off-4 : b.off]
// v = v[:4]
// low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24)
MOVQ bitReaderShifted_in(br{{ var "id" }}), AX
MOVL 0(br_offset)(AX*1), AX // AX = uint32(b.in[b.off:b.off+4])
// b.value |= uint64(low) << (b.bitsRead & 63)
MOVQ br_bits_read, CX
SHLQ CL, AX
ORQ AX, br_value
// exhausted = exhausted || (br{{ var "id"}}.off < 4)
CMPQ br_offset, $4
SETLT DL
ORB DL, DH
// }
skip_fill{{ var "id" }}:
// val0 := br{{ var "id"}}.peekTopBits(peekBits)
MOVQ br_value, AX
MOVQ peek_bits, CX
SHRQ CL, AX // AX = (value >> peek_bits) & mask
// v0 := table[val0&mask]
MOVW 0(table)(AX*2), AX // AX - v0
// br{{ var "id"}}.advance(uint8(v0.entry))
MOVB AH, BL // BL = uint8(v0.entry >> 8)
MOVBQZX AL, CX
SHLQ CL, br_value // value <<= n
ADDQ CX, br_bits_read // bits_read += n
// val1 := br{{ var "id"}}.peekTopBits(peekBits)
MOVQ peek_bits, CX
MOVQ br_value, AX
SHRQ CL, AX // AX = (value >> peek_bits) & mask
// v1 := table[val1&mask]
MOVW 0(table)(AX*2), AX // AX - v1
// br{{ var "id"}}.advance(uint8(v1.entry))
MOVB AH, BH // BH = uint8(v1.entry >> 8)
MOVBQZX AL, CX
SHLQ CX, br_value // value <<= n
ADDQ CX, br_bits_read // bits_read += n
// these two writes get coalesced
// buf[stream][off] = uint8(v0.entry >> 8)
// buf[stream][off+1] = uint8(v1.entry >> 8)
MOVW BX, {{ var "bufofs" }}(buffer)(off*1)
// SECOND PART:
// val2 := br{{ var "id"}}.peekTopBits(peekBits)
MOVQ br_value, AX
MOVQ peek_bits, CX
SHRQ CL, AX // AX = (value >> peek_bits) & mask
// v2 := table[val0&mask]
MOVW 0(table)(AX*2), AX // AX - v0
// br{{ var "id"}}.advance(uint8(v0.entry))
MOVB AH, BL // BL = uint8(v0.entry >> 8)
MOVBQZX AL, CX
SHLQ CL, br_value // value <<= n
ADDQ CX, br_bits_read // bits_read += n
// val3 := br{{ var "id"}}.peekTopBits(peekBits)
MOVQ peek_bits, CX
MOVQ br_value, AX
SHRQ CL, AX // AX = (value >> peek_bits) & mask
// v3 := table[val1&mask]
MOVW 0(table)(AX*2), AX // AX - v1
// br{{ var "id"}}.advance(uint8(v1.entry))
MOVB AH, BH // BH = uint8(v1.entry >> 8)
MOVBQZX AL, CX
SHLQ CX, br_value // value <<= n
ADDQ CX, br_bits_read // bits_read += n
// these two writes get coalesced
// buf[stream][off+2] = uint8(v2.entry >> 8)
// buf[stream][off+3] = uint8(v3.entry >> 8)
MOVW BX, {{ var "bufofs" }}+2(buffer)(off*1)
// update the bitrader reader structure
MOVB br_bits_read, bitReaderShifted_bitsRead(br{{ var "id" }})
MOVQ br_value, bitReaderShifted_value(br{{ var "id" }})
MOVQ br_offset, bitReaderShifted_off(br{{ var "id" }})
{{ end }}
{{ set "id" "0" }}
{{ set "ofs" "0" }}
{{ set "bufofs" "0" }} {{/* id * bufoff */}}
{{ template "decode_2_values_x86" . }}
{{ set "id" "1" }}
{{ set "ofs" "8" }}
{{ set "bufofs" "256" }}
{{ template "decode_2_values_x86" . }}
{{ set "id" "2" }}
{{ set "ofs" "16" }}
{{ set "bufofs" "512" }}
{{ template "decode_2_values_x86" . }}
{{ set "id" "3" }}
{{ set "ofs" "24" }}
{{ set "bufofs" "768" }}
{{ template "decode_2_values_x86" . }}
ADDQ $4, off // off += 2
TESTB DH, DH // any br[i].ofs < 4?
JNZ end
CMPQ off, $bufoff
JL main_loop
end:
MOVQ 0(SP), BP
MOVB off, ret+56(FP)
RET
#undef off
#undef buffer
#undef table
#undef br_bits_read
#undef br_value
#undef br_offset
#undef peek_bits
#undef exhausted
#undef br0
#undef br1
#undef br2
#undef br3

181
vendor/github.com/klauspost/compress/huff0/decompress_amd64.go generated vendored Normal file
View File

@ -0,0 +1,181 @@
//go:build amd64 && !appengine && !noasm && gc
// +build amd64,!appengine,!noasm,gc
// This file contains the specialisation of Decoder.Decompress4X
// that uses an asm implementation of its main loop.
package huff0
import (
"errors"
"fmt"
)
// decompress4x_main_loop_x86 is an x86 assembler implementation
// of Decompress4X when tablelog > 8.
// go:noescape
func decompress4x_main_loop_x86(pbr0, pbr1, pbr2, pbr3 *bitReaderShifted,
peekBits uint8, buf *byte, tbl *dEntrySingle) uint8
// decompress4x_8b_loop_x86 is an x86 assembler implementation
// of Decompress4X when tablelog <= 8 which decodes 4 entries
// per loop.
// go:noescape
func decompress4x_8b_loop_x86(pbr0, pbr1, pbr2, pbr3 *bitReaderShifted,
peekBits uint8, buf *byte, tbl *dEntrySingle) uint8
// fallback8BitSize is the size where using Go version is faster.
const fallback8BitSize = 800
// Decompress4X will decompress a 4X encoded stream.
// The length of the supplied input must match the end of a block exactly.
// The *capacity* of the dst slice must match the destination size of
// the uncompressed data exactly.
func (d *Decoder) Decompress4X(dst, src []byte) ([]byte, error) {
if len(d.dt.single) == 0 {
return nil, errors.New("no table loaded")
}
if len(src) < 6+(4*1) {
return nil, errors.New("input too small")
}
use8BitTables := d.actualTableLog <= 8
if cap(dst) < fallback8BitSize && use8BitTables {
return d.decompress4X8bit(dst, src)
}
var br [4]bitReaderShifted
// Decode "jump table"
start := 6
for i := 0; i < 3; i++ {
length := int(src[i*2]) | (int(src[i*2+1]) << 8)
if start+length >= len(src) {
return nil, errors.New("truncated input (or invalid offset)")
}
err := br[i].init(src[start : start+length])
if err != nil {
return nil, err
}
start += length
}
err := br[3].init(src[start:])
if err != nil {
return nil, err
}
// destination, offset to match first output
dstSize := cap(dst)
dst = dst[:dstSize]
out := dst
dstEvery := (dstSize + 3) / 4
const tlSize = 1 << tableLogMax
const tlMask = tlSize - 1
single := d.dt.single[:tlSize]
// Use temp table to avoid bound checks/append penalty.
buf := d.buffer()
var off uint8
var decoded int
const debug = false
// see: bitReaderShifted.peekBitsFast()
peekBits := uint8((64 - d.actualTableLog) & 63)
// Decode 2 values from each decoder/loop.
const bufoff = 256
for {
if br[0].off < 4 || br[1].off < 4 || br[2].off < 4 || br[3].off < 4 {
break
}
if use8BitTables {
off = decompress4x_8b_loop_x86(&br[0], &br[1], &br[2], &br[3], peekBits, &buf[0][0], &single[0])
} else {
off = decompress4x_main_loop_x86(&br[0], &br[1], &br[2], &br[3], peekBits, &buf[0][0], &single[0])
}
if debug {
fmt.Print("DEBUG: ")
fmt.Printf("off=%d,", off)
for i := 0; i < 4; i++ {
fmt.Printf(" br[%d]={bitsRead=%d, value=%x, off=%d}",
i, br[i].bitsRead, br[i].value, br[i].off)
}
fmt.Println("")
}
if off != 0 {
break
}
if bufoff > dstEvery {
d.bufs.Put(buf)
return nil, errors.New("corruption detected: stream overrun 1")
}
copy(out, buf[0][:])
copy(out[dstEvery:], buf[1][:])
copy(out[dstEvery*2:], buf[2][:])
copy(out[dstEvery*3:], buf[3][:])
out = out[bufoff:]
decoded += bufoff * 4
// There must at least be 3 buffers left.
if len(out) < dstEvery*3 {
d.bufs.Put(buf)
return nil, errors.New("corruption detected: stream overrun 2")
}
}
if off > 0 {
ioff := int(off)
if len(out) < dstEvery*3+ioff {
d.bufs.Put(buf)
return nil, errors.New("corruption detected: stream overrun 3")
}
copy(out, buf[0][:off])
copy(out[dstEvery:], buf[1][:off])
copy(out[dstEvery*2:], buf[2][:off])
copy(out[dstEvery*3:], buf[3][:off])
decoded += int(off) * 4
out = out[off:]
}
// Decode remaining.
remainBytes := dstEvery - (decoded / 4)
for i := range br {
offset := dstEvery * i
endsAt := offset + remainBytes
if endsAt > len(out) {
endsAt = len(out)
}
br := &br[i]
bitsLeft := br.remaining()
for bitsLeft > 0 {
br.fill()
if offset >= endsAt {
d.bufs.Put(buf)
return nil, errors.New("corruption detected: stream overrun 4")
}
// Read value and increment offset.
val := br.peekBitsFast(d.actualTableLog)
v := single[val&tlMask].entry
nBits := uint8(v)
br.advance(nBits)
bitsLeft -= uint(nBits)
out[offset] = uint8(v >> 8)
offset++
}
if offset != endsAt {
d.bufs.Put(buf)
return nil, fmt.Errorf("corruption detected: short output block %d, end %d != %d", i, offset, endsAt)
}
decoded += offset - dstEvery*i
err = br.close()
if err != nil {
return nil, err
}
}
d.bufs.Put(buf)
if dstSize != decoded {
return nil, errors.New("corruption detected: short output block")
}
return dst, nil
}

506
vendor/github.com/klauspost/compress/huff0/decompress_amd64.s generated vendored Normal file
View File

@ -0,0 +1,506 @@
// +build !appengine
// +build gc
// +build !noasm
#include "textflag.h"
#include "funcdata.h"
#include "go_asm.h"
#ifdef GOAMD64_v4
#ifndef GOAMD64_v3
#define GOAMD64_v3
#endif
#endif
#define bufoff 256 // see decompress.go, we're using [4][256]byte table
// func decompress4x_main_loop_x86(pbr0, pbr1, pbr2, pbr3 *bitReaderShifted,
// peekBits uint8, buf *byte, tbl *dEntrySingle) (int, bool)
TEXT ·decompress4x_main_loop_x86(SB), NOSPLIT, $8
#define off R8
#define buffer DI
#define table SI
#define br_bits_read R9
#define br_value R10
#define br_offset R11
#define peek_bits R12
#define exhausted DX
#define br0 R13
#define br1 R14
#define br2 R15
#define br3 BP
MOVQ BP, 0(SP)
XORQ exhausted, exhausted // exhausted = false
XORQ off, off // off = 0
MOVBQZX peekBits+32(FP), peek_bits
MOVQ buf+40(FP), buffer
MOVQ tbl+48(FP), table
MOVQ pbr0+0(FP), br0
MOVQ pbr1+8(FP), br1
MOVQ pbr2+16(FP), br2
MOVQ pbr3+24(FP), br3
main_loop:
// const stream = 0
// br0.fillFast()
MOVBQZX bitReaderShifted_bitsRead(br0), br_bits_read
MOVQ bitReaderShifted_value(br0), br_value
MOVQ bitReaderShifted_off(br0), br_offset
// We must have at least 2 * max tablelog left
CMPQ br_bits_read, $64-22
JBE skip_fill0
SUBQ $32, br_bits_read // b.bitsRead -= 32
SUBQ $4, br_offset // b.off -= 4
// v := b.in[b.off-4 : b.off]
// v = v[:4]
// low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24)
MOVQ bitReaderShifted_in(br0), AX
// b.value |= uint64(low) << (b.bitsRead & 63)
#ifdef GOAMD64_v3
SHLXQ br_bits_read, 0(br_offset)(AX*1), AX // AX = uint32(b.in[b.off:b.off+4]) << (b.bitsRead & 63)
#else
MOVL 0(br_offset)(AX*1), AX // AX = uint32(b.in[b.off:b.off+4])
MOVQ br_bits_read, CX
SHLQ CL, AX
#endif
ORQ AX, br_value
// exhausted = exhausted || (br0.off < 4)
CMPQ br_offset, $4
SETLT DL
ORB DL, DH
// }
skip_fill0:
// val0 := br0.peekTopBits(peekBits)
#ifdef GOAMD64_v3
SHRXQ peek_bits, br_value, AX // AX = (value >> peek_bits) & mask
#else
MOVQ br_value, AX
MOVQ peek_bits, CX
SHRQ CL, AX // AX = (value >> peek_bits) & mask
#endif
// v0 := table[val0&mask]
MOVW 0(table)(AX*2), AX // AX - v0
// br0.advance(uint8(v0.entry))
MOVB AH, BL // BL = uint8(v0.entry >> 8)
#ifdef GOAMD64_v3
MOVBQZX AL, CX
SHLXQ AX, br_value, br_value // value <<= n
#else
MOVBQZX AL, CX
SHLQ CL, br_value // value <<= n
#endif
ADDQ CX, br_bits_read // bits_read += n
#ifdef GOAMD64_v3
SHRXQ peek_bits, br_value, AX // AX = (value >> peek_bits) & mask
#else
// val1 := br0.peekTopBits(peekBits)
MOVQ peek_bits, CX
MOVQ br_value, AX
SHRQ CL, AX // AX = (value >> peek_bits) & mask
#endif
// v1 := table[val1&mask]
MOVW 0(table)(AX*2), AX // AX - v1
// br0.advance(uint8(v1.entry))
MOVB AH, BH // BH = uint8(v1.entry >> 8)
#ifdef GOAMD64_v3
MOVBQZX AL, CX
SHLXQ AX, br_value, br_value // value <<= n
#else
MOVBQZX AL, CX
SHLQ CL, br_value // value <<= n
#endif
ADDQ CX, br_bits_read // bits_read += n
// these two writes get coalesced
// buf[stream][off] = uint8(v0.entry >> 8)
// buf[stream][off+1] = uint8(v1.entry >> 8)
MOVW BX, 0(buffer)(off*1)
// update the bitrader reader structure
MOVB br_bits_read, bitReaderShifted_bitsRead(br0)
MOVQ br_value, bitReaderShifted_value(br0)
MOVQ br_offset, bitReaderShifted_off(br0)
// const stream = 1
// br1.fillFast()
MOVBQZX bitReaderShifted_bitsRead(br1), br_bits_read
MOVQ bitReaderShifted_value(br1), br_value
MOVQ bitReaderShifted_off(br1), br_offset
// We must have at least 2 * max tablelog left
CMPQ br_bits_read, $64-22
JBE skip_fill1
SUBQ $32, br_bits_read // b.bitsRead -= 32
SUBQ $4, br_offset // b.off -= 4
// v := b.in[b.off-4 : b.off]
// v = v[:4]
// low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24)
MOVQ bitReaderShifted_in(br1), AX
// b.value |= uint64(low) << (b.bitsRead & 63)
#ifdef GOAMD64_v3
SHLXQ br_bits_read, 0(br_offset)(AX*1), AX // AX = uint32(b.in[b.off:b.off+4]) << (b.bitsRead & 63)
#else
MOVL 0(br_offset)(AX*1), AX // AX = uint32(b.in[b.off:b.off+4])
MOVQ br_bits_read, CX
SHLQ CL, AX
#endif
ORQ AX, br_value
// exhausted = exhausted || (br1.off < 4)
CMPQ br_offset, $4
SETLT DL
ORB DL, DH
// }
skip_fill1:
// val0 := br1.peekTopBits(peekBits)
#ifdef GOAMD64_v3
SHRXQ peek_bits, br_value, AX // AX = (value >> peek_bits) & mask
#else
MOVQ br_value, AX
MOVQ peek_bits, CX
SHRQ CL, AX // AX = (value >> peek_bits) & mask
#endif
// v0 := table[val0&mask]
MOVW 0(table)(AX*2), AX // AX - v0
// br1.advance(uint8(v0.entry))
MOVB AH, BL // BL = uint8(v0.entry >> 8)
#ifdef GOAMD64_v3
MOVBQZX AL, CX
SHLXQ AX, br_value, br_value // value <<= n
#else
MOVBQZX AL, CX
SHLQ CL, br_value // value <<= n
#endif
ADDQ CX, br_bits_read // bits_read += n
#ifdef GOAMD64_v3
SHRXQ peek_bits, br_value, AX // AX = (value >> peek_bits) & mask
#else
// val1 := br1.peekTopBits(peekBits)
MOVQ peek_bits, CX
MOVQ br_value, AX
SHRQ CL, AX // AX = (value >> peek_bits) & mask
#endif
// v1 := table[val1&mask]
MOVW 0(table)(AX*2), AX // AX - v1
// br1.advance(uint8(v1.entry))
MOVB AH, BH // BH = uint8(v1.entry >> 8)
#ifdef GOAMD64_v3
MOVBQZX AL, CX
SHLXQ AX, br_value, br_value // value <<= n
#else
MOVBQZX AL, CX
SHLQ CL, br_value // value <<= n
#endif
ADDQ CX, br_bits_read // bits_read += n
// these two writes get coalesced
// buf[stream][off] = uint8(v0.entry >> 8)
// buf[stream][off+1] = uint8(v1.entry >> 8)
MOVW BX, 256(buffer)(off*1)
// update the bitrader reader structure
MOVB br_bits_read, bitReaderShifted_bitsRead(br1)
MOVQ br_value, bitReaderShifted_value(br1)
MOVQ br_offset, bitReaderShifted_off(br1)
// const stream = 2
// br2.fillFast()
MOVBQZX bitReaderShifted_bitsRead(br2), br_bits_read
MOVQ bitReaderShifted_value(br2), br_value
MOVQ bitReaderShifted_off(br2), br_offset
// We must have at least 2 * max tablelog left
CMPQ br_bits_read, $64-22
JBE skip_fill2
SUBQ $32, br_bits_read // b.bitsRead -= 32
SUBQ $4, br_offset // b.off -= 4
// v := b.in[b.off-4 : b.off]
// v = v[:4]
// low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24)
MOVQ bitReaderShifted_in(br2), AX
// b.value |= uint64(low) << (b.bitsRead & 63)
#ifdef GOAMD64_v3
SHLXQ br_bits_read, 0(br_offset)(AX*1), AX // AX = uint32(b.in[b.off:b.off+4]) << (b.bitsRead & 63)
#else
MOVL 0(br_offset)(AX*1), AX // AX = uint32(b.in[b.off:b.off+4])
MOVQ br_bits_read, CX
SHLQ CL, AX
#endif
ORQ AX, br_value
// exhausted = exhausted || (br2.off < 4)
CMPQ br_offset, $4
SETLT DL
ORB DL, DH
// }
skip_fill2:
// val0 := br2.peekTopBits(peekBits)
#ifdef GOAMD64_v3
SHRXQ peek_bits, br_value, AX // AX = (value >> peek_bits) & mask
#else
MOVQ br_value, AX
MOVQ peek_bits, CX
SHRQ CL, AX // AX = (value >> peek_bits) & mask
#endif
// v0 := table[val0&mask]
MOVW 0(table)(AX*2), AX // AX - v0
// br2.advance(uint8(v0.entry))
MOVB AH, BL // BL = uint8(v0.entry >> 8)
#ifdef GOAMD64_v3
MOVBQZX AL, CX
SHLXQ AX, br_value, br_value // value <<= n
#else
MOVBQZX AL, CX
SHLQ CL, br_value // value <<= n
#endif
ADDQ CX, br_bits_read // bits_read += n
#ifdef GOAMD64_v3
SHRXQ peek_bits, br_value, AX // AX = (value >> peek_bits) & mask
#else
// val1 := br2.peekTopBits(peekBits)
MOVQ peek_bits, CX
MOVQ br_value, AX
SHRQ CL, AX // AX = (value >> peek_bits) & mask
#endif
// v1 := table[val1&mask]
MOVW 0(table)(AX*2), AX // AX - v1
// br2.advance(uint8(v1.entry))
MOVB AH, BH // BH = uint8(v1.entry >> 8)
#ifdef GOAMD64_v3
MOVBQZX AL, CX
SHLXQ AX, br_value, br_value // value <<= n
#else
MOVBQZX AL, CX
SHLQ CL, br_value // value <<= n
#endif
ADDQ CX, br_bits_read // bits_read += n
// these two writes get coalesced
// buf[stream][off] = uint8(v0.entry >> 8)
// buf[stream][off+1] = uint8(v1.entry >> 8)
MOVW BX, 512(buffer)(off*1)
// update the bitrader reader structure
MOVB br_bits_read, bitReaderShifted_bitsRead(br2)
MOVQ br_value, bitReaderShifted_value(br2)
MOVQ br_offset, bitReaderShifted_off(br2)
// const stream = 3
// br3.fillFast()
MOVBQZX bitReaderShifted_bitsRead(br3), br_bits_read
MOVQ bitReaderShifted_value(br3), br_value
MOVQ bitReaderShifted_off(br3), br_offset
// We must have at least 2 * max tablelog left
CMPQ br_bits_read, $64-22
JBE skip_fill3
SUBQ $32, br_bits_read // b.bitsRead -= 32
SUBQ $4, br_offset // b.off -= 4
// v := b.in[b.off-4 : b.off]
// v = v[:4]
// low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24)
MOVQ bitReaderShifted_in(br3), AX
// b.value |= uint64(low) << (b.bitsRead & 63)
#ifdef GOAMD64_v3
SHLXQ br_bits_read, 0(br_offset)(AX*1), AX // AX = uint32(b.in[b.off:b.off+4]) << (b.bitsRead & 63)
#else
MOVL 0(br_offset)(AX*1), AX // AX = uint32(b.in[b.off:b.off+4])
MOVQ br_bits_read, CX
SHLQ CL, AX
#endif
ORQ AX, br_value
// exhausted = exhausted || (br3.off < 4)
CMPQ br_offset, $4
SETLT DL
ORB DL, DH
// }
skip_fill3:
// val0 := br3.peekTopBits(peekBits)
#ifdef GOAMD64_v3
SHRXQ peek_bits, br_value, AX // AX = (value >> peek_bits) & mask
#else
MOVQ br_value, AX
MOVQ peek_bits, CX
SHRQ CL, AX // AX = (value >> peek_bits) & mask
#endif
// v0 := table[val0&mask]
MOVW 0(table)(AX*2), AX // AX - v0
// br3.advance(uint8(v0.entry))
MOVB AH, BL // BL = uint8(v0.entry >> 8)
#ifdef GOAMD64_v3
MOVBQZX AL, CX
SHLXQ AX, br_value, br_value // value <<= n
#else
MOVBQZX AL, CX
SHLQ CL, br_value // value <<= n
#endif
ADDQ CX, br_bits_read // bits_read += n
#ifdef GOAMD64_v3
SHRXQ peek_bits, br_value, AX // AX = (value >> peek_bits) & mask
#else
// val1 := br3.peekTopBits(peekBits)
MOVQ peek_bits, CX
MOVQ br_value, AX
SHRQ CL, AX // AX = (value >> peek_bits) & mask
#endif
// v1 := table[val1&mask]
MOVW 0(table)(AX*2), AX // AX - v1
// br3.advance(uint8(v1.entry))
MOVB AH, BH // BH = uint8(v1.entry >> 8)
#ifdef GOAMD64_v3
MOVBQZX AL, CX
SHLXQ AX, br_value, br_value // value <<= n
#else
MOVBQZX AL, CX
SHLQ CL, br_value // value <<= n
#endif
ADDQ CX, br_bits_read // bits_read += n
// these two writes get coalesced
// buf[stream][off] = uint8(v0.entry >> 8)
// buf[stream][off+1] = uint8(v1.entry >> 8)
MOVW BX, 768(buffer)(off*1)
// update the bitrader reader structure
MOVB br_bits_read, bitReaderShifted_bitsRead(br3)
MOVQ br_value, bitReaderShifted_value(br3)
MOVQ br_offset, bitReaderShifted_off(br3)
ADDQ $2, off // off += 2
TESTB DH, DH // any br[i].ofs < 4?
JNZ end
CMPQ off, $bufoff
JL main_loop
end:
MOVQ 0(SP), BP
MOVB off, ret+56(FP)
RET
#undef off
#undef buffer
#undef table
#undef br_bits_read
#undef br_value
#undef br_offset
#undef peek_bits
#undef exhausted
#undef br0
#undef br1
#undef br2
#undef br3

195
vendor/github.com/klauspost/compress/huff0/decompress_amd64.s.in generated vendored Normal file
View File

@ -0,0 +1,195 @@
// +build !appengine
// +build gc
// +build !noasm
#include "textflag.h"
#include "funcdata.h"
#include "go_asm.h"
#ifdef GOAMD64_v4
#ifndef GOAMD64_v3
#define GOAMD64_v3
#endif
#endif
#define bufoff 256 // see decompress.go, we're using [4][256]byte table
//func decompress4x_main_loop_x86(pbr0, pbr1, pbr2, pbr3 *bitReaderShifted,
// peekBits uint8, buf *byte, tbl *dEntrySingle) (int, bool)
TEXT ·decompress4x_main_loop_x86(SB), NOSPLIT, $8
#define off R8
#define buffer DI
#define table SI
#define br_bits_read R9
#define br_value R10
#define br_offset R11
#define peek_bits R12
#define exhausted DX
#define br0 R13
#define br1 R14
#define br2 R15
#define br3 BP
MOVQ BP, 0(SP)
XORQ exhausted, exhausted // exhausted = false
XORQ off, off // off = 0
MOVBQZX peekBits+32(FP), peek_bits
MOVQ buf+40(FP), buffer
MOVQ tbl+48(FP), table
MOVQ pbr0+0(FP), br0
MOVQ pbr1+8(FP), br1
MOVQ pbr2+16(FP), br2
MOVQ pbr3+24(FP), br3
main_loop:
{{ define "decode_2_values_x86" }}
// const stream = {{ var "id" }}
// br{{ var "id"}}.fillFast()
MOVBQZX bitReaderShifted_bitsRead(br{{ var "id" }}), br_bits_read
MOVQ bitReaderShifted_value(br{{ var "id" }}), br_value
MOVQ bitReaderShifted_off(br{{ var "id" }}), br_offset
// We must have at least 2 * max tablelog left
CMPQ br_bits_read, $64-22
JBE skip_fill{{ var "id" }}
SUBQ $32, br_bits_read // b.bitsRead -= 32
SUBQ $4, br_offset // b.off -= 4
// v := b.in[b.off-4 : b.off]
// v = v[:4]
// low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24)
MOVQ bitReaderShifted_in(br{{ var "id" }}), AX
// b.value |= uint64(low) << (b.bitsRead & 63)
#ifdef GOAMD64_v3
SHLXQ br_bits_read, 0(br_offset)(AX*1), AX // AX = uint32(b.in[b.off:b.off+4]) << (b.bitsRead & 63)
#else
MOVL 0(br_offset)(AX*1), AX // AX = uint32(b.in[b.off:b.off+4])
MOVQ br_bits_read, CX
SHLQ CL, AX
#endif
ORQ AX, br_value
// exhausted = exhausted || (br{{ var "id"}}.off < 4)
CMPQ br_offset, $4
SETLT DL
ORB DL, DH
// }
skip_fill{{ var "id" }}:
// val0 := br{{ var "id"}}.peekTopBits(peekBits)
#ifdef GOAMD64_v3
SHRXQ peek_bits, br_value, AX // AX = (value >> peek_bits) & mask
#else
MOVQ br_value, AX
MOVQ peek_bits, CX
SHRQ CL, AX // AX = (value >> peek_bits) & mask
#endif
// v0 := table[val0&mask]
MOVW 0(table)(AX*2), AX // AX - v0
// br{{ var "id"}}.advance(uint8(v0.entry))
MOVB AH, BL // BL = uint8(v0.entry >> 8)
#ifdef GOAMD64_v3
MOVBQZX AL, CX
SHLXQ AX, br_value, br_value // value <<= n
#else
MOVBQZX AL, CX
SHLQ CL, br_value // value <<= n
#endif
ADDQ CX, br_bits_read // bits_read += n
#ifdef GOAMD64_v3
SHRXQ peek_bits, br_value, AX // AX = (value >> peek_bits) & mask
#else
// val1 := br{{ var "id"}}.peekTopBits(peekBits)
MOVQ peek_bits, CX
MOVQ br_value, AX
SHRQ CL, AX // AX = (value >> peek_bits) & mask
#endif
// v1 := table[val1&mask]
MOVW 0(table)(AX*2), AX // AX - v1
// br{{ var "id"}}.advance(uint8(v1.entry))
MOVB AH, BH // BH = uint8(v1.entry >> 8)
#ifdef GOAMD64_v3
MOVBQZX AL, CX
SHLXQ AX, br_value, br_value // value <<= n
#else
MOVBQZX AL, CX
SHLQ CL, br_value // value <<= n
#endif
ADDQ CX, br_bits_read // bits_read += n
// these two writes get coalesced
// buf[stream][off] = uint8(v0.entry >> 8)
// buf[stream][off+1] = uint8(v1.entry >> 8)
MOVW BX, {{ var "bufofs" }}(buffer)(off*1)
// update the bitrader reader structure
MOVB br_bits_read, bitReaderShifted_bitsRead(br{{ var "id" }})
MOVQ br_value, bitReaderShifted_value(br{{ var "id" }})
MOVQ br_offset, bitReaderShifted_off(br{{ var "id" }})
{{ end }}
{{ set "id" "0" }}
{{ set "ofs" "0" }}
{{ set "bufofs" "0" }} {{/* id * bufoff */}}
{{ template "decode_2_values_x86" . }}
{{ set "id" "1" }}
{{ set "ofs" "8" }}
{{ set "bufofs" "256" }}
{{ template "decode_2_values_x86" . }}
{{ set "id" "2" }}
{{ set "ofs" "16" }}
{{ set "bufofs" "512" }}
{{ template "decode_2_values_x86" . }}
{{ set "id" "3" }}
{{ set "ofs" "24" }}
{{ set "bufofs" "768" }}
{{ template "decode_2_values_x86" . }}
ADDQ $2, off // off += 2
TESTB DH, DH // any br[i].ofs < 4?
JNZ end
CMPQ off, $bufoff
JL main_loop
end:
MOVQ 0(SP), BP
MOVB off, ret+56(FP)
RET
#undef off
#undef buffer
#undef table
#undef br_bits_read
#undef br_value
#undef br_offset
#undef peek_bits
#undef exhausted
#undef br0
#undef br1
#undef br2
#undef br3

193
vendor/github.com/klauspost/compress/huff0/decompress_generic.go generated vendored Normal file
View File

@ -0,0 +1,193 @@
//go:build !amd64 || appengine || !gc || noasm
// +build !amd64 appengine !gc noasm
// This file contains a generic implementation of Decoder.Decompress4X.
package huff0
import (
"errors"
"fmt"
)
// Decompress4X will decompress a 4X encoded stream.
// The length of the supplied input must match the end of a block exactly.
// The *capacity* of the dst slice must match the destination size of
// the uncompressed data exactly.
func (d *Decoder) Decompress4X(dst, src []byte) ([]byte, error) {
if len(d.dt.single) == 0 {
return nil, errors.New("no table loaded")
}
if len(src) < 6+(4*1) {
return nil, errors.New("input too small")
}
if use8BitTables && d.actualTableLog <= 8 {
return d.decompress4X8bit(dst, src)
}
var br [4]bitReaderShifted
// Decode "jump table"
start := 6
for i := 0; i < 3; i++ {
length := int(src[i*2]) | (int(src[i*2+1]) << 8)
if start+length >= len(src) {
return nil, errors.New("truncated input (or invalid offset)")
}
err := br[i].init(src[start : start+length])
if err != nil {
return nil, err
}
start += length
}
err := br[3].init(src[start:])
if err != nil {
return nil, err
}
// destination, offset to match first output
dstSize := cap(dst)
dst = dst[:dstSize]
out := dst
dstEvery := (dstSize + 3) / 4
const tlSize = 1 << tableLogMax
const tlMask = tlSize - 1
single := d.dt.single[:tlSize]
// Use temp table to avoid bound checks/append penalty.
buf := d.buffer()
var off uint8
var decoded int
// Decode 2 values from each decoder/loop.
const bufoff = 256
for {
if br[0].off < 4 || br[1].off < 4 || br[2].off < 4 || br[3].off < 4 {
break
}
{
const stream = 0
const stream2 = 1
br[stream].fillFast()
br[stream2].fillFast()
val := br[stream].peekBitsFast(d.actualTableLog)
val2 := br[stream2].peekBitsFast(d.actualTableLog)
v := single[val&tlMask]
v2 := single[val2&tlMask]
br[stream].advance(uint8(v.entry))
br[stream2].advance(uint8(v2.entry))
buf[stream][off] = uint8(v.entry >> 8)
buf[stream2][off] = uint8(v2.entry >> 8)
val = br[stream].peekBitsFast(d.actualTableLog)
val2 = br[stream2].peekBitsFast(d.actualTableLog)
v = single[val&tlMask]
v2 = single[val2&tlMask]
br[stream].advance(uint8(v.entry))
br[stream2].advance(uint8(v2.entry))
buf[stream][off+1] = uint8(v.entry >> 8)
buf[stream2][off+1] = uint8(v2.entry >> 8)
}
{
const stream = 2
const stream2 = 3
br[stream].fillFast()
br[stream2].fillFast()
val := br[stream].peekBitsFast(d.actualTableLog)
val2 := br[stream2].peekBitsFast(d.actualTableLog)
v := single[val&tlMask]
v2 := single[val2&tlMask]
br[stream].advance(uint8(v.entry))
br[stream2].advance(uint8(v2.entry))
buf[stream][off] = uint8(v.entry >> 8)
buf[stream2][off] = uint8(v2.entry >> 8)
val = br[stream].peekBitsFast(d.actualTableLog)
val2 = br[stream2].peekBitsFast(d.actualTableLog)
v = single[val&tlMask]
v2 = single[val2&tlMask]
br[stream].advance(uint8(v.entry))
br[stream2].advance(uint8(v2.entry))
buf[stream][off+1] = uint8(v.entry >> 8)
buf[stream2][off+1] = uint8(v2.entry >> 8)
}
off += 2
if off == 0 {
if bufoff > dstEvery {
d.bufs.Put(buf)
return nil, errors.New("corruption detected: stream overrun 1")
}
copy(out, buf[0][:])
copy(out[dstEvery:], buf[1][:])
copy(out[dstEvery*2:], buf[2][:])
copy(out[dstEvery*3:], buf[3][:])
out = out[bufoff:]
decoded += bufoff * 4
// There must at least be 3 buffers left.
if len(out) < dstEvery*3 {
d.bufs.Put(buf)
return nil, errors.New("corruption detected: stream overrun 2")
}
}
}
if off > 0 {
ioff := int(off)
if len(out) < dstEvery*3+ioff {
d.bufs.Put(buf)
return nil, errors.New("corruption detected: stream overrun 3")
}
copy(out, buf[0][:off])
copy(out[dstEvery:], buf[1][:off])
copy(out[dstEvery*2:], buf[2][:off])
copy(out[dstEvery*3:], buf[3][:off])
decoded += int(off) * 4
out = out[off:]
}
// Decode remaining.
remainBytes := dstEvery - (decoded / 4)
for i := range br {
offset := dstEvery * i
endsAt := offset + remainBytes
if endsAt > len(out) {
endsAt = len(out)
}
br := &br[i]
bitsLeft := br.remaining()
for bitsLeft > 0 {
br.fill()
if offset >= endsAt {
d.bufs.Put(buf)
return nil, errors.New("corruption detected: stream overrun 4")
}
// Read value and increment offset.
val := br.peekBitsFast(d.actualTableLog)
v := single[val&tlMask].entry
nBits := uint8(v)
br.advance(nBits)
bitsLeft -= uint(nBits)
out[offset] = uint8(v >> 8)
offset++
}
if offset != endsAt {
d.bufs.Put(buf)
return nil, fmt.Errorf("corruption detected: short output block %d, end %d != %d", i, offset, endsAt)
}
decoded += offset - dstEvery*i
err = br.close()
if err != nil {
return nil, err
}
}
d.bufs.Put(buf)
if dstSize != decoded {
return nil, errors.New("corruption detected: short output block")
}
return dst, nil
}

2
vendor/github.com/klauspost/compress/huff0/huff0.go generated vendored
View File

@ -8,6 +8,7 @@ import (
"fmt" "fmt"
"math" "math"
"math/bits" "math/bits"
"sync"
"github.com/klauspost/compress/fse" "github.com/klauspost/compress/fse"
) )
@ -116,6 +117,7 @@ type Scratch struct {
nodes []nodeElt nodes []nodeElt
tmpOut [4][]byte tmpOut [4][]byte
fse *fse.Scratch fse *fse.Scratch
decPool sync.Pool // *[4][256]byte buffers.
huffWeight [maxSymbolValue + 1]byte huffWeight [maxSymbolValue + 1]byte
} }

4
vendor/github.com/klauspost/compress/s2/encodeblock_amd64.go generated vendored
View File

@ -1,7 +1,7 @@
// Code generated by command: go run gen.go -out ../encodeblock_amd64.s -stubs ../encodeblock_amd64.go -pkg=s2. DO NOT EDIT. // Code generated by command: go run gen.go -out ../encodeblock_amd64.s -stubs ../encodeblock_amd64.go -pkg=s2. DO NOT EDIT.
//go:build !appengine && !noasm && gc //go:build !appengine && !noasm && gc && !noasm
// +build !appengine,!noasm,gc // +build !appengine,!noasm,gc,!noasm
package s2 package s2

1503
vendor/github.com/klauspost/compress/s2/encodeblock_amd64.s generated vendored
View File

File diff suppressed because it is too large Load Diff

106
vendor/github.com/klauspost/compress/zstd/README.md generated vendored
View File

@ -78,6 +78,9 @@ of a stream. This is independent of the `WithEncoderConcurrency(n)`, but that is
in the future. So if you want to limit concurrency for future updates, specify the concurrency in the future. So if you want to limit concurrency for future updates, specify the concurrency
you would like. you would like.
If you would like stream encoding to be done without spawning async goroutines, use `WithEncoderConcurrency(1)`
which will compress input as each block is completed, blocking on writes until each has completed.
You can specify your desired compression level using `WithEncoderLevel()` option. Currently only pre-defined You can specify your desired compression level using `WithEncoderLevel()` option. Currently only pre-defined
compression settings can be specified. compression settings can be specified.
@ -104,7 +107,8 @@ and seems to ignore concatenated streams, even though [it is part of the spec](h
For compressing small blocks, the returned encoder has a function called `EncodeAll(src, dst []byte) []byte`. For compressing small blocks, the returned encoder has a function called `EncodeAll(src, dst []byte) []byte`.
`EncodeAll` will encode all input in src and append it to dst. `EncodeAll` will encode all input in src and append it to dst.
This function can be called concurrently, but each call will only run on a single goroutine. This function can be called concurrently.
Each call will only run on a same goroutine as the caller.
Encoded blocks can be concatenated and the result will be the combined input stream. Encoded blocks can be concatenated and the result will be the combined input stream.
Data compressed with EncodeAll can be decoded with the Decoder, using either a stream or `DecodeAll`. Data compressed with EncodeAll can be decoded with the Decoder, using either a stream or `DecodeAll`.
@ -149,10 +153,10 @@ http://sun.aei.polsl.pl/~sdeor/corpus/silesia.zip
This package: This package:
file out level insize outsize millis mb/s file out level insize outsize millis mb/s
silesia.tar zskp 1 211947520 73101992 643 313.87 silesia.tar zskp 1 211947520 73821326 634 318.47
silesia.tar zskp 2 211947520 67504318 969 208.38 silesia.tar zskp 2 211947520 67655404 1508 133.96
silesia.tar zskp 3 211947520 64595893 2007 100.68 silesia.tar zskp 3 211947520 64746933 3000 67.37
silesia.tar zskp 4 211947520 60995370 8825 22.90 silesia.tar zskp 4 211947520 60073508 16926 11.94
cgo zstd: cgo zstd:
silesia.tar zstd 1 211947520 73605392 543 371.56 silesia.tar zstd 1 211947520 73605392 543 371.56
@ -161,94 +165,94 @@ silesia.tar zstd 6 211947520 62916450 1913 105.66
silesia.tar zstd 9 211947520 60212393 5063 39.92 silesia.tar zstd 9 211947520 60212393 5063 39.92
gzip, stdlib/this package: gzip, stdlib/this package:
silesia.tar gzstd 1 211947520 80007735 1654 122.21 silesia.tar gzstd 1 211947520 80007735 1498 134.87
silesia.tar gzkp 1 211947520 80136201 1152 175.45 silesia.tar gzkp 1 211947520 80088272 1009 200.31
GOB stream of binary data. Highly compressible. GOB stream of binary data. Highly compressible.
https://files.klauspost.com/compress/gob-stream.7z https://files.klauspost.com/compress/gob-stream.7z
file out level insize outsize millis mb/s file out level insize outsize millis mb/s
gob-stream zskp 1 1911399616 235022249 3088 590.30 gob-stream zskp 1 1911399616 233948096 3230 564.34
gob-stream zskp 2 1911399616 205669791 3786 481.34 gob-stream zskp 2 1911399616 203997694 4997 364.73
gob-stream zskp 3 1911399616 175034659 9636 189.17 gob-stream zskp 3 1911399616 173526523 13435 135.68
gob-stream zskp 4 1911399616 165609838 50369 36.19 gob-stream zskp 4 1911399616 162195235 47559 38.33
gob-stream zstd 1 1911399616 249810424 2637 691.26 gob-stream zstd 1 1911399616 249810424 2637 691.26
gob-stream zstd 3 1911399616 208192146 3490 522.31 gob-stream zstd 3 1911399616 208192146 3490 522.31
gob-stream zstd 6 1911399616 193632038 6687 272.56 gob-stream zstd 6 1911399616 193632038 6687 272.56
gob-stream zstd 9 1911399616 177620386 16175 112.70 gob-stream zstd 9 1911399616 177620386 16175 112.70
gob-stream gzstd 1 1911399616 357382641 10251 177.82 gob-stream gzstd 1 1911399616 357382013 9046 201.49
gob-stream gzkp 1 1911399616 359753026 5438 335.20 gob-stream gzkp 1 1911399616 359136669 4885 373.08
The test data for the Large Text Compression Benchmark is the first The test data for the Large Text Compression Benchmark is the first
10^9 bytes of the English Wikipedia dump on Mar. 3, 2006. 10^9 bytes of the English Wikipedia dump on Mar. 3, 2006.
http://mattmahoney.net/dc/textdata.html http://mattmahoney.net/dc/textdata.html
file out level insize outsize millis mb/s file out level insize outsize millis mb/s
enwik9 zskp 1 1000000000 343848582 3609 264.18 enwik9 zskp 1 1000000000 343833605 3687 258.64
enwik9 zskp 2 1000000000 317276632 5746 165.97 enwik9 zskp 2 1000000000 317001237 7672 124.29
enwik9 zskp 3 1000000000 292243069 12162 78.41 enwik9 zskp 3 1000000000 291915823 15923 59.89
enwik9 zskp 4 1000000000 262183768 82837 11.51 enwik9 zskp 4 1000000000 261710291 77697 12.27
enwik9 zstd 1 1000000000 358072021 3110 306.65 enwik9 zstd 1 1000000000 358072021 3110 306.65
enwik9 zstd 3 1000000000 313734672 4784 199.35 enwik9 zstd 3 1000000000 313734672 4784 199.35
enwik9 zstd 6 1000000000 295138875 10290 92.68 enwik9 zstd 6 1000000000 295138875 10290 92.68
enwik9 zstd 9 1000000000 278348700 28549 33.40 enwik9 zstd 9 1000000000 278348700 28549 33.40
enwik9 gzstd 1 1000000000 382578136 9604 99.30 enwik9 gzstd 1 1000000000 382578136 8608 110.78
enwik9 gzkp 1 1000000000 383825945 6544 145.73 enwik9 gzkp 1 1000000000 382781160 5628 169.45
Highly compressible JSON file. Highly compressible JSON file.
https://files.klauspost.com/compress/github-june-2days-2019.json.zst https://files.klauspost.com/compress/github-june-2days-2019.json.zst
file out level insize outsize millis mb/s file out level insize outsize millis mb/s
github-june-2days-2019.json zskp 1 6273951764 699045015 10620 563.40 github-june-2days-2019.json zskp 1 6273951764 697439532 9789 611.17
github-june-2days-2019.json zskp 2 6273951764 617881763 11687 511.96 github-june-2days-2019.json zskp 2 6273951764 610876538 18553 322.49
github-june-2days-2019.json zskp 3 6273951764 524340691 34043 175.75 github-june-2days-2019.json zskp 3 6273951764 517662858 44186 135.41
github-june-2days-2019.json zskp 4 6273951764 470320075 170190 35.16 github-june-2days-2019.json zskp 4 6273951764 464617114 165373 36.18
github-june-2days-2019.json zstd 1 6273951764 766284037 8450 708.00 github-june-2days-2019.json zstd 1 6273951764 766284037 8450 708.00
github-june-2days-2019.json zstd 3 6273951764 661889476 10927 547.57 github-june-2days-2019.json zstd 3 6273951764 661889476 10927 547.57
github-june-2days-2019.json zstd 6 6273951764 642756859 22996 260.18 github-june-2days-2019.json zstd 6 6273951764 642756859 22996 260.18
github-june-2days-2019.json zstd 9 6273951764 601974523 52413 114.16 github-june-2days-2019.json zstd 9 6273951764 601974523 52413 114.16
github-june-2days-2019.json gzstd 1 6273951764 1164400847 29948 199.79 github-june-2days-2019.json gzstd 1 6273951764 1164397768 26793 223.32
github-june-2days-2019.json gzkp 1 6273951764 1125417694 21788 274.61 github-june-2days-2019.json gzkp 1 6273951764 1120631856 17693 338.16
VM Image, Linux mint with a few installed applications: VM Image, Linux mint with a few installed applications:
https://files.klauspost.com/compress/rawstudio-mint14.7z https://files.klauspost.com/compress/rawstudio-mint14.7z
file out level insize outsize millis mb/s file out level insize outsize millis mb/s
rawstudio-mint14.tar zskp 1 8558382592 3667489370 20210 403.84 rawstudio-mint14.tar zskp 1 8558382592 3718400221 18206 448.29
rawstudio-mint14.tar zskp 2 8558382592 3364592300 31873 256.07 rawstudio-mint14.tar zskp 2 8558382592 3326118337 37074 220.15
rawstudio-mint14.tar zskp 3 8558382592 3158085214 77675 105.08 rawstudio-mint14.tar zskp 3 8558382592 3163842361 87306 93.49
rawstudio-mint14.tar zskp 4 8558382592 2965110639 857750 9.52 rawstudio-mint14.tar zskp 4 8558382592 2970480650 783862 10.41
rawstudio-mint14.tar zstd 1 8558382592 3609250104 17136 476.27 rawstudio-mint14.tar zstd 1 8558382592 3609250104 17136 476.27
rawstudio-mint14.tar zstd 3 8558382592 3341679997 29262 278.92 rawstudio-mint14.tar zstd 3 8558382592 3341679997 29262 278.92
rawstudio-mint14.tar zstd 6 8558382592 3235846406 77904 104.77 rawstudio-mint14.tar zstd 6 8558382592 3235846406 77904 104.77
rawstudio-mint14.tar zstd 9 8558382592 3160778861 140946 57.91 rawstudio-mint14.tar zstd 9 8558382592 3160778861 140946 57.91
rawstudio-mint14.tar gzstd 1 8558382592 3926257486 57722 141.40 rawstudio-mint14.tar gzstd 1 8558382592 3926234992 51345 158.96
rawstudio-mint14.tar gzkp 1 8558382592 3962605659 45113 180.92 rawstudio-mint14.tar gzkp 1 8558382592 3960117298 36722 222.26
CSV data: CSV data:
https://files.klauspost.com/compress/nyc-taxi-data-10M.csv.zst https://files.klauspost.com/compress/nyc-taxi-data-10M.csv.zst
file out level insize outsize millis mb/s file out level insize outsize millis mb/s
nyc-taxi-data-10M.csv zskp 1 3325605752 641339945 8925 355.35 nyc-taxi-data-10M.csv zskp 1 3325605752 641319332 9462 335.17
nyc-taxi-data-10M.csv zskp 2 3325605752 591748091 11268 281.44 nyc-taxi-data-10M.csv zskp 2 3325605752 588976126 17570 180.50
nyc-taxi-data-10M.csv zskp 3 3325605752 530289687 25239 125.66 nyc-taxi-data-10M.csv zskp 3 3325605752 529329260 32432 97.79
nyc-taxi-data-10M.csv zskp 4 3325605752 476268884 135958 23.33 nyc-taxi-data-10M.csv zskp 4 3325605752 474949772 138025 22.98
nyc-taxi-data-10M.csv zstd 1 3325605752 687399637 8233 385.18 nyc-taxi-data-10M.csv zstd 1 3325605752 687399637 8233 385.18
nyc-taxi-data-10M.csv zstd 3 3325605752 598514411 10065 315.07 nyc-taxi-data-10M.csv zstd 3 3325605752 598514411 10065 315.07
nyc-taxi-data-10M.csv zstd 6 3325605752 570522953 20038 158.27 nyc-taxi-data-10M.csv zstd 6 3325605752 570522953 20038 158.27
nyc-taxi-data-10M.csv zstd 9 3325605752 517554797 64565 49.12 nyc-taxi-data-10M.csv zstd 9 3325605752 517554797 64565 49.12
nyc-taxi-data-10M.csv gzstd 1 3325605752 928656485 23876 132.83 nyc-taxi-data-10M.csv gzstd 1 3325605752 928654908 21270 149.11
nyc-taxi-data-10M.csv gzkp 1 3325605752 922257165 16780 189.00 nyc-taxi-data-10M.csv gzkp 1 3325605752 922273214 13929 227.68
``` ```
## Decompressor ## Decompressor
@ -283,8 +287,13 @@ func Decompress(in io.Reader, out io.Writer) error {
} }
``` ```
It is important to use the "Close" function when you no longer need the Reader to stop running goroutines. It is important to use the "Close" function when you no longer need the Reader to stop running goroutines,
See "Allocation-less operation" below. when running with default settings.
Goroutines will exit once an error has been returned, including `io.EOF` at the end of a stream.
Streams are decoded concurrently in 4 asynchronous stages to give the best possible throughput.
However, if you prefer synchronous decompression, use `WithDecoderConcurrency(1)` which will decompress data
as it is being requested only.
For decoding buffers, it could look something like this: For decoding buffers, it could look something like this:
@ -293,7 +302,7 @@ import "github.com/klauspost/compress/zstd"
// Create a reader that caches decompressors. // Create a reader that caches decompressors.
// For this operation type we supply a nil Reader. // For this operation type we supply a nil Reader.
var decoder, _ = zstd.NewReader(nil) var decoder, _ = zstd.NewReader(nil, WithDecoderConcurrency(0))
// Decompress a buffer. We don't supply a destination buffer, // Decompress a buffer. We don't supply a destination buffer,
// so it will be allocated by the decoder. // so it will be allocated by the decoder.
@ -304,8 +313,11 @@ func Decompress(src []byte) ([]byte, error) {
Both of these cases should provide the functionality needed. Both of these cases should provide the functionality needed.
The decoder can be used for *concurrent* decompression of multiple buffers. The decoder can be used for *concurrent* decompression of multiple buffers.
By default 4 decompressors will be created.
It will only allow a certain number of concurrent operations to run. It will only allow a certain number of concurrent operations to run.
To tweak that yourself use the `WithDecoderConcurrency(n)` option when creating the decoder. To tweak that yourself use the `WithDecoderConcurrency(n)` option when creating the decoder.
It is possible to use `WithDecoderConcurrency(0)` to create GOMAXPROCS decoders.
### Dictionaries ### Dictionaries
@ -357,18 +369,20 @@ In this case no unneeded allocations should be made.
The buffer decoder does everything on the same goroutine and does nothing concurrently. The buffer decoder does everything on the same goroutine and does nothing concurrently.
It can however decode several buffers concurrently. Use `WithDecoderConcurrency(n)` to limit that. It can however decode several buffers concurrently. Use `WithDecoderConcurrency(n)` to limit that.
The stream decoder operates on The stream decoder will create goroutines that:
* One goroutine reads input and splits the input to several block decoders. 1) Reads input and splits the input into blocks.
* A number of decoders will decode blocks. 2) Decompression of literals.
* A goroutine coordinates these blocks and sends history from one to the next. 3) Decompression of sequences.
4) Reconstruction of output stream.
So effectively this also means the decoder will "read ahead" and prepare data to always be available for output. So effectively this also means the decoder will "read ahead" and prepare data to always be available for output.
The concurrency level will, for streams, determine how many blocks ahead the compression will start.
Since "blocks" are quite dependent on the output of the previous block stream decoding will only have limited concurrency. Since "blocks" are quite dependent on the output of the previous block stream decoding will only have limited concurrency.
In practice this means that concurrency is often limited to utilizing about 2 cores effectively. In practice this means that concurrency is often limited to utilizing about 3 cores effectively.
### Benchmarks ### Benchmarks

4
vendor/github.com/klauspost/compress/zstd/bitreader.go generated vendored
View File

@ -7,6 +7,7 @@ package zstd
import ( import (
"encoding/binary" "encoding/binary"
"errors" "errors"
"fmt"
"io" "io"
"math/bits" "math/bits"
) )
@ -132,6 +133,9 @@ func (b *bitReader) remain() uint {
func (b *bitReader) close() error { func (b *bitReader) close() error {
// Release reference. // Release reference.
b.in = nil b.in = nil
if !b.finished() {
return fmt.Errorf("%d extra bits on block, should be 0", b.remain())
}
if b.bitsRead > 64 { if b.bitsRead > 64 {
return io.ErrUnexpectedEOF return io.ErrUnexpectedEOF
} }

424
vendor/github.com/klauspost/compress/zstd/blockdec.go generated vendored
View File

@ -76,16 +76,25 @@ type blockDec struct {
// Window size of the block. // Window size of the block.
WindowSize uint64 WindowSize uint64
history chan *history
input chan struct{}
result chan decodeOutput
err error err error
decWG sync.WaitGroup
// Check against this crc
checkCRC []byte
// Frame to use for singlethreaded decoding. // Frame to use for singlethreaded decoding.
// Should not be used by the decoder itself since parent may be another frame. // Should not be used by the decoder itself since parent may be another frame.
localFrame *frameDec localFrame *frameDec
sequence []seqVals
async struct {
newHist *history
literals []byte
seqData []byte
seqSize int // Size of uncompressed sequences
fcs uint64
}
// Block is RLE, this is the size. // Block is RLE, this is the size.
RLESize uint32 RLESize uint32
tmp [4]byte tmp [4]byte
@ -109,12 +118,7 @@ func (b *blockDec) String() string {
func newBlockDec(lowMem bool) *blockDec { func newBlockDec(lowMem bool) *blockDec {
b := blockDec{ b := blockDec{
lowMem: lowMem, lowMem: lowMem,
result: make(chan decodeOutput, 1),
input: make(chan struct{}, 1),
history: make(chan *history, 1),
} }
b.decWG.Add(1)
go b.startDecoder()
return &b return &b
} }
@ -137,6 +141,12 @@ func (b *blockDec) reset(br byteBuffer, windowSize uint64) error {
case blockTypeReserved: case blockTypeReserved:
return ErrReservedBlockType return ErrReservedBlockType
case blockTypeRLE: case blockTypeRLE:
if cSize > maxCompressedBlockSize || cSize > int(b.WindowSize) {
if debugDecoder {
printf("rle block too big: csize:%d block: %+v\n", uint64(cSize), b)
}
return ErrWindowSizeExceeded
}
b.RLESize = uint32(cSize) b.RLESize = uint32(cSize)
if b.lowMem { if b.lowMem {
maxSize = cSize maxSize = cSize
@ -157,7 +167,19 @@ func (b *blockDec) reset(br byteBuffer, windowSize uint64) error {
} }
return ErrCompressedSizeTooBig return ErrCompressedSizeTooBig
} }
// Empty compressed blocks must at least be 2 bytes
// for Literals_Block_Type and one for Sequences_Section_Header.
if cSize < 2 {
return ErrBlockTooSmall
}
case blockTypeRaw: case blockTypeRaw:
if cSize > maxCompressedBlockSize || cSize > int(b.WindowSize) {
if debugDecoder {
printf("rle block too big: csize:%d block: %+v\n", uint64(cSize), b)
}
return ErrWindowSizeExceeded
}
b.RLESize = 0 b.RLESize = 0
// We do not need a destination for raw blocks. // We do not need a destination for raw blocks.
maxSize = -1 maxSize = -1
@ -192,85 +214,14 @@ func (b *blockDec) sendErr(err error) {
b.Last = true b.Last = true
b.Type = blockTypeReserved b.Type = blockTypeReserved
b.err = err b.err = err
b.input <- struct{}{}
} }
// Close will release resources. // Close will release resources.
// Closed blockDec cannot be reset. // Closed blockDec cannot be reset.
func (b *blockDec) Close() { func (b *blockDec) Close() {
close(b.input)
close(b.history)
close(b.result)
b.decWG.Wait()
} }
// decodeAsync will prepare decoding the block when it receives input. // decodeBuf
// This will separate output and history.
func (b *blockDec) startDecoder() {
defer b.decWG.Done()
for range b.input {
//println("blockDec: Got block input")
switch b.Type {
case blockTypeRLE:
if cap(b.dst) < int(b.RLESize) {
if b.lowMem {
b.dst = make([]byte, b.RLESize)
} else {
b.dst = make([]byte, maxBlockSize)
}
}
o := decodeOutput{
d: b,
b: b.dst[:b.RLESize],
err: nil,
}
v := b.data[0]
for i := range o.b {
o.b[i] = v
}
hist := <-b.history
hist.append(o.b)
b.result <- o
case blockTypeRaw:
o := decodeOutput{
d: b,
b: b.data,
err: nil,
}
hist := <-b.history
hist.append(o.b)
b.result <- o
case blockTypeCompressed:
b.dst = b.dst[:0]
err := b.decodeCompressed(nil)
o := decodeOutput{
d: b,
b: b.dst,
err: err,
}
if debugDecoder {
println("Decompressed to", len(b.dst), "bytes, error:", err)
}
b.result <- o
case blockTypeReserved:
// Used for returning errors.
<-b.history
b.result <- decodeOutput{
d: b,
b: nil,
err: b.err,
}
default:
panic("Invalid block type")
}
if debugDecoder {
println("blockDec: Finished block")
}
}
}
// decodeAsync will prepare decoding the block when it receives the history.
// If history is provided, it will not fetch it from the channel.
func (b *blockDec) decodeBuf(hist *history) error { func (b *blockDec) decodeBuf(hist *history) error {
switch b.Type { switch b.Type {
case blockTypeRLE: case blockTypeRLE:
@ -293,14 +244,23 @@ func (b *blockDec) decodeBuf(hist *history) error {
return nil return nil
case blockTypeCompressed: case blockTypeCompressed:
saved := b.dst saved := b.dst
// Append directly to history
if hist.ignoreBuffer == 0 {
b.dst = hist.b b.dst = hist.b
hist.b = nil hist.b = nil
} else {
b.dst = b.dst[:0]
}
err := b.decodeCompressed(hist) err := b.decodeCompressed(hist)
if debugDecoder { if debugDecoder {
println("Decompressed to total", len(b.dst), "bytes, hash:", xxhash.Sum64(b.dst), "error:", err) println("Decompressed to total", len(b.dst), "bytes, hash:", xxhash.Sum64(b.dst), "error:", err)
} }
if hist.ignoreBuffer == 0 {
hist.b = b.dst hist.b = b.dst
b.dst = saved b.dst = saved
} else {
hist.appendKeep(b.dst)
}
return err return err
case blockTypeReserved: case blockTypeReserved:
// Used for returning errors. // Used for returning errors.
@ -310,30 +270,18 @@ func (b *blockDec) decodeBuf(hist *history) error {
} }
} }
// decodeCompressed will start decompressing a block. func (b *blockDec) decodeLiterals(in []byte, hist *history) (remain []byte, err error) {
// If no history is supplied the decoder will decodeAsync as much as possible
// before fetching from blockDec.history
func (b *blockDec) decodeCompressed(hist *history) error {
in := b.data
delayedHistory := hist == nil
if delayedHistory {
// We must always grab history.
defer func() {
if hist == nil {
<-b.history
}
}()
}
// There must be at least one byte for Literals_Block_Type and one for Sequences_Section_Header // There must be at least one byte for Literals_Block_Type and one for Sequences_Section_Header
if len(in) < 2 { if len(in) < 2 {
return ErrBlockTooSmall return in, ErrBlockTooSmall
} }
litType := literalsBlockType(in[0] & 3) litType := literalsBlockType(in[0] & 3)
var litRegenSize int var litRegenSize int
var litCompSize int var litCompSize int
sizeFormat := (in[0] >> 2) & 3 sizeFormat := (in[0] >> 2) & 3
var fourStreams bool var fourStreams bool
var literals []byte
switch litType { switch litType {
case literalsBlockRaw, literalsBlockRLE: case literalsBlockRaw, literalsBlockRLE:
switch sizeFormat { switch sizeFormat {
@ -349,7 +297,7 @@ func (b *blockDec) decodeCompressed(hist *history) error {
// Regenerated_Size uses 20 bits (0-1048575). Literals_Section_Header uses 3 bytes. // Regenerated_Size uses 20 bits (0-1048575). Literals_Section_Header uses 3 bytes.
if len(in) < 3 { if len(in) < 3 {
println("too small: litType:", litType, " sizeFormat", sizeFormat, len(in)) println("too small: litType:", litType, " sizeFormat", sizeFormat, len(in))
return ErrBlockTooSmall return in, ErrBlockTooSmall
} }
litRegenSize = int(in[0]>>4) + (int(in[1]) << 4) + (int(in[2]) << 12) litRegenSize = int(in[0]>>4) + (int(in[1]) << 4) + (int(in[2]) << 12)
in = in[3:] in = in[3:]
@ -360,7 +308,7 @@ func (b *blockDec) decodeCompressed(hist *history) error {
// Both Regenerated_Size and Compressed_Size use 10 bits (0-1023). // Both Regenerated_Size and Compressed_Size use 10 bits (0-1023).
if len(in) < 3 { if len(in) < 3 {
println("too small: litType:", litType, " sizeFormat", sizeFormat, len(in)) println("too small: litType:", litType, " sizeFormat", sizeFormat, len(in))
return ErrBlockTooSmall return in, ErrBlockTooSmall
} }
n := uint64(in[0]>>4) + (uint64(in[1]) << 4) + (uint64(in[2]) << 12) n := uint64(in[0]>>4) + (uint64(in[1]) << 4) + (uint64(in[2]) << 12)
litRegenSize = int(n & 1023) litRegenSize = int(n & 1023)
@ -371,7 +319,7 @@ func (b *blockDec) decodeCompressed(hist *history) error {
fourStreams = true fourStreams = true
if len(in) < 4 { if len(in) < 4 {
println("too small: litType:", litType, " sizeFormat", sizeFormat, len(in)) println("too small: litType:", litType, " sizeFormat", sizeFormat, len(in))
return ErrBlockTooSmall return in, ErrBlockTooSmall
} }
n := uint64(in[0]>>4) + (uint64(in[1]) << 4) + (uint64(in[2]) << 12) + (uint64(in[3]) << 20) n := uint64(in[0]>>4) + (uint64(in[1]) << 4) + (uint64(in[2]) << 12) + (uint64(in[3]) << 20)
litRegenSize = int(n & 16383) litRegenSize = int(n & 16383)
@ -381,7 +329,7 @@ func (b *blockDec) decodeCompressed(hist *history) error {
fourStreams = true fourStreams = true
if len(in) < 5 { if len(in) < 5 {
println("too small: litType:", litType, " sizeFormat", sizeFormat, len(in)) println("too small: litType:", litType, " sizeFormat", sizeFormat, len(in))
return ErrBlockTooSmall return in, ErrBlockTooSmall
} }
n := uint64(in[0]>>4) + (uint64(in[1]) << 4) + (uint64(in[2]) << 12) + (uint64(in[3]) << 20) + (uint64(in[4]) << 28) n := uint64(in[0]>>4) + (uint64(in[1]) << 4) + (uint64(in[2]) << 12) + (uint64(in[3]) << 20) + (uint64(in[4]) << 28)
litRegenSize = int(n & 262143) litRegenSize = int(n & 262143)
@ -392,13 +340,15 @@ func (b *blockDec) decodeCompressed(hist *history) error {
if debugDecoder { if debugDecoder {
println("literals type:", litType, "litRegenSize:", litRegenSize, "litCompSize:", litCompSize, "sizeFormat:", sizeFormat, "4X:", fourStreams) println("literals type:", litType, "litRegenSize:", litRegenSize, "litCompSize:", litCompSize, "sizeFormat:", sizeFormat, "4X:", fourStreams)
} }
var literals []byte if litRegenSize > int(b.WindowSize) || litRegenSize > maxCompressedBlockSize {
var huff *huff0.Scratch return in, ErrWindowSizeExceeded
}
switch litType { switch litType {
case literalsBlockRaw: case literalsBlockRaw:
if len(in) < litRegenSize { if len(in) < litRegenSize {
println("too small: litType:", litType, " sizeFormat", sizeFormat, "remain:", len(in), "want:", litRegenSize) println("too small: litType:", litType, " sizeFormat", sizeFormat, "remain:", len(in), "want:", litRegenSize)
return ErrBlockTooSmall return in, ErrBlockTooSmall
} }
literals = in[:litRegenSize] literals = in[:litRegenSize]
in = in[litRegenSize:] in = in[litRegenSize:]
@ -406,7 +356,7 @@ func (b *blockDec) decodeCompressed(hist *history) error {
case literalsBlockRLE: case literalsBlockRLE:
if len(in) < 1 { if len(in) < 1 {
println("too small: litType:", litType, " sizeFormat", sizeFormat, "remain:", len(in), "want:", 1) println("too small: litType:", litType, " sizeFormat", sizeFormat, "remain:", len(in), "want:", 1)
return ErrBlockTooSmall return in, ErrBlockTooSmall
} }
if cap(b.literalBuf) < litRegenSize { if cap(b.literalBuf) < litRegenSize {
if b.lowMem { if b.lowMem {
@ -417,7 +367,6 @@ func (b *blockDec) decodeCompressed(hist *history) error {
b.literalBuf = make([]byte, litRegenSize) b.literalBuf = make([]byte, litRegenSize)
} else { } else {
b.literalBuf = make([]byte, litRegenSize, maxCompressedLiteralSize) b.literalBuf = make([]byte, litRegenSize, maxCompressedLiteralSize)
} }
} }
} }
@ -433,7 +382,7 @@ func (b *blockDec) decodeCompressed(hist *history) error {
case literalsBlockTreeless: case literalsBlockTreeless:
if len(in) < litCompSize { if len(in) < litCompSize {
println("too small: litType:", litType, " sizeFormat", sizeFormat, "remain:", len(in), "want:", litCompSize) println("too small: litType:", litType, " sizeFormat", sizeFormat, "remain:", len(in), "want:", litCompSize)
return ErrBlockTooSmall return in, ErrBlockTooSmall
} }
// Store compressed literals, so we defer decoding until we get history. // Store compressed literals, so we defer decoding until we get history.
literals = in[:litCompSize] literals = in[:litCompSize]
@ -441,15 +390,10 @@ func (b *blockDec) decodeCompressed(hist *history) error {
if debugDecoder { if debugDecoder {
printf("Found %d compressed literals\n", litCompSize) printf("Found %d compressed literals\n", litCompSize)
} }
case literalsBlockCompressed: huff := hist.huffTree
if len(in) < litCompSize { if huff == nil {
println("too small: litType:", litType, " sizeFormat", sizeFormat, "remain:", len(in), "want:", litCompSize) return in, errors.New("literal block was treeless, but no history was defined")
return ErrBlockTooSmall
} }
literals = in[:litCompSize]
in = in[litCompSize:]
huff = huffDecoderPool.Get().(*huff0.Scratch)
var err error
// Ensure we have space to store it. // Ensure we have space to store it.
if cap(b.literalBuf) < litRegenSize { if cap(b.literalBuf) < litRegenSize {
if b.lowMem { if b.lowMem {
@ -458,14 +402,53 @@ func (b *blockDec) decodeCompressed(hist *history) error {
b.literalBuf = make([]byte, 0, maxCompressedLiteralSize) b.literalBuf = make([]byte, 0, maxCompressedLiteralSize)
} }
} }
var err error
// Use our out buffer.
huff.MaxDecodedSize = maxCompressedBlockSize
if fourStreams {
literals, err = huff.Decoder().Decompress4X(b.literalBuf[:0:litRegenSize], literals)
} else {
literals, err = huff.Decoder().Decompress1X(b.literalBuf[:0:litRegenSize], literals)
}
// Make sure we don't leak our literals buffer
if err != nil {
println("decompressing literals:", err)
return in, err
}
if len(literals) != litRegenSize {
return in, fmt.Errorf("literal output size mismatch want %d, got %d", litRegenSize, len(literals))
}
case literalsBlockCompressed:
if len(in) < litCompSize {
println("too small: litType:", litType, " sizeFormat", sizeFormat, "remain:", len(in), "want:", litCompSize)
return in, ErrBlockTooSmall
}
literals = in[:litCompSize]
in = in[litCompSize:]
// Ensure we have space to store it.
if cap(b.literalBuf) < litRegenSize {
if b.lowMem {
b.literalBuf = make([]byte, 0, litRegenSize)
} else {
b.literalBuf = make([]byte, 0, maxCompressedBlockSize)
}
}
huff := hist.huffTree
if huff == nil || (hist.dict != nil && huff == hist.dict.litEnc) {
huff = huffDecoderPool.Get().(*huff0.Scratch)
if huff == nil { if huff == nil {
huff = &huff0.Scratch{} huff = &huff0.Scratch{}
} }
}
var err error
huff, literals, err = huff0.ReadTable(literals, huff) huff, literals, err = huff0.ReadTable(literals, huff)
if err != nil { if err != nil {
println("reading huffman table:", err) println("reading huffman table:", err)
return err return in, err
} }
hist.huffTree = huff
huff.MaxDecodedSize = maxCompressedBlockSize
// Use our out buffer. // Use our out buffer.
if fourStreams { if fourStreams {
literals, err = huff.Decoder().Decompress4X(b.literalBuf[:0:litRegenSize], literals) literals, err = huff.Decoder().Decompress4X(b.literalBuf[:0:litRegenSize], literals)
@ -474,27 +457,56 @@ func (b *blockDec) decodeCompressed(hist *history) error {
} }
if err != nil { if err != nil {
println("decoding compressed literals:", err) println("decoding compressed literals:", err)
return err return in, err
} }
// Make sure we don't leak our literals buffer // Make sure we don't leak our literals buffer
if len(literals) != litRegenSize { if len(literals) != litRegenSize {
return fmt.Errorf("literal output size mismatch want %d, got %d", litRegenSize, len(literals)) return in, fmt.Errorf("literal output size mismatch want %d, got %d", litRegenSize, len(literals))
} }
if debugDecoder { if debugDecoder {
printf("Decompressed %d literals into %d bytes\n", litCompSize, litRegenSize) printf("Decompressed %d literals into %d bytes\n", litCompSize, litRegenSize)
} }
} }
hist.decoders.literals = literals
return in, nil
}
// decodeCompressed will start decompressing a block.
func (b *blockDec) decodeCompressed(hist *history) error {
in := b.data
in, err := b.decodeLiterals(in, hist)
if err != nil {
return err
}
err = b.prepareSequences(in, hist)
if err != nil {
return err
}
if hist.decoders.nSeqs == 0 {
b.dst = append(b.dst, hist.decoders.literals...)
return nil
}
err = hist.decoders.decodeSync(hist)
if err != nil {
return err
}
b.dst = hist.decoders.out
hist.recentOffsets = hist.decoders.prevOffset
return nil
}
func (b *blockDec) prepareSequences(in []byte, hist *history) (err error) {
if debugDecoder {
printf("prepareSequences: %d byte(s) input\n", len(in))
}
// Decode Sequences // Decode Sequences
// https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#sequences-section // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#sequences-section
if len(in) < 1 { if len(in) < 1 {
return ErrBlockTooSmall return ErrBlockTooSmall
} }
var nSeqs int
seqHeader := in[0] seqHeader := in[0]
nSeqs := 0
switch { switch {
case seqHeader == 0:
in = in[1:]
case seqHeader < 128: case seqHeader < 128:
nSeqs = int(seqHeader) nSeqs = int(seqHeader)
in = in[1:] in = in[1:]
@ -511,8 +523,16 @@ func (b *blockDec) decodeCompressed(hist *history) error {
nSeqs = 0x7f00 + int(in[1]) + (int(in[2]) << 8) nSeqs = 0x7f00 + int(in[1]) + (int(in[2]) << 8)
in = in[3:] in = in[3:]
} }
if nSeqs == 0 && len(in) != 0 {
// When no sequences, there should not be any more data...
if debugDecoder {
printf("prepareSequences: 0 sequences, but %d byte(s) left on stream\n", len(in))
}
return ErrUnexpectedBlockSize
}
var seqs = &sequenceDecs{} var seqs = &hist.decoders
seqs.nSeqs = nSeqs
if nSeqs > 0 { if nSeqs > 0 {
if len(in) < 1 { if len(in) < 1 {
return ErrBlockTooSmall return ErrBlockTooSmall
@ -541,6 +561,9 @@ func (b *blockDec) decodeCompressed(hist *history) error {
} }
switch mode { switch mode {
case compModePredefined: case compModePredefined:
if seq.fse != nil && !seq.fse.preDefined {
fseDecoderPool.Put(seq.fse)
}
seq.fse = &fsePredef[i] seq.fse = &fsePredef[i]
case compModeRLE: case compModeRLE:
if br.remain() < 1 { if br.remain() < 1 {
@ -548,34 +571,36 @@ func (b *blockDec) decodeCompressed(hist *history) error {
} }
v := br.Uint8() v := br.Uint8()
br.advance(1) br.advance(1)
dec := fseDecoderPool.Get().(*fseDecoder) if seq.fse == nil || seq.fse.preDefined {
seq.fse = fseDecoderPool.Get().(*fseDecoder)
}
symb, err := decSymbolValue(v, symbolTableX[i]) symb, err := decSymbolValue(v, symbolTableX[i])
if err != nil { if err != nil {
printf("RLE Transform table (%v) error: %v", tableIndex(i), err) printf("RLE Transform table (%v) error: %v", tableIndex(i), err)
return err return err
} }
dec.setRLE(symb) seq.fse.setRLE(symb)
seq.fse = dec
if debugDecoder { if debugDecoder {
printf("RLE set to %+v, code: %v", symb, v) printf("RLE set to %+v, code: %v", symb, v)
} }
case compModeFSE: case compModeFSE:
println("Reading table for", tableIndex(i)) println("Reading table for", tableIndex(i))
dec := fseDecoderPool.Get().(*fseDecoder) if seq.fse == nil || seq.fse.preDefined {
err := dec.readNCount(&br, uint16(maxTableSymbol[i])) seq.fse = fseDecoderPool.Get().(*fseDecoder)
}
err := seq.fse.readNCount(&br, uint16(maxTableSymbol[i]))
if err != nil { if err != nil {
println("Read table error:", err) println("Read table error:", err)
return err return err
} }
err = dec.transform(symbolTableX[i]) err = seq.fse.transform(symbolTableX[i])
if err != nil { if err != nil {
println("Transform table error:", err) println("Transform table error:", err)
return err return err
} }
if debugDecoder { if debugDecoder {
println("Read table ok", "symbolLen:", dec.symbolLen) println("Read table ok", "symbolLen:", seq.fse.symbolLen)
} }
seq.fse = dec
case compModeRepeat: case compModeRepeat:
seq.repeat = true seq.repeat = true
} }
@ -585,91 +610,52 @@ func (b *blockDec) decodeCompressed(hist *history) error {
} }
in = br.unread() in = br.unread()
} }
// Wait for history.
// All time spent after this is critical since it is strictly sequential.
if hist == nil {
hist = <-b.history
if hist.error {
return ErrDecoderClosed
}
}
// Decode treeless literal block.
if litType == literalsBlockTreeless {
// TODO: We could send the history early WITHOUT the stream history.
// This would allow decoding treeless literals before the byte history is available.
// Silencia stats: Treeless 4393, with: 32775, total: 37168, 11% treeless.
// So not much obvious gain here.
if hist.huffTree == nil {
return errors.New("literal block was treeless, but no history was defined")
}
// Ensure we have space to store it.
if cap(b.literalBuf) < litRegenSize {
if b.lowMem {
b.literalBuf = make([]byte, 0, litRegenSize)
} else {
b.literalBuf = make([]byte, 0, maxCompressedLiteralSize)
}
}
var err error
// Use our out buffer.
huff = hist.huffTree
if fourStreams {
literals, err = huff.Decoder().Decompress4X(b.literalBuf[:0:litRegenSize], literals)
} else {
literals, err = huff.Decoder().Decompress1X(b.literalBuf[:0:litRegenSize], literals)
}
// Make sure we don't leak our literals buffer
if err != nil {
println("decompressing literals:", err)
return err
}
if len(literals) != litRegenSize {
return fmt.Errorf("literal output size mismatch want %d, got %d", litRegenSize, len(literals))
}
} else {
if hist.huffTree != nil && huff != nil {
if hist.dict == nil || hist.dict.litEnc != hist.huffTree {
huffDecoderPool.Put(hist.huffTree)
}
hist.huffTree = nil
}
}
if huff != nil {
hist.huffTree = huff
}
if debugDecoder { if debugDecoder {
println("Final literals:", len(literals), "hash:", xxhash.Sum64(literals), "and", nSeqs, "sequences.") println("Literals:", len(seqs.literals), "hash:", xxhash.Sum64(seqs.literals), "and", seqs.nSeqs, "sequences.")
} }
if nSeqs == 0 { if nSeqs == 0 {
// Decompressed content is defined entirely as Literals Section content. if len(b.sequence) > 0 {
b.dst = append(b.dst, literals...) b.sequence = b.sequence[:0]
if delayedHistory {
hist.append(literals)
} }
return nil return nil
} }
br := seqs.br
seqs, err := seqs.mergeHistory(&hist.decoders) if br == nil {
if err != nil { br = &bitReader{}
return err
} }
if debugDecoder {
println("History merged ok")
}
br := &bitReader{}
if err := br.init(in); err != nil { if err := br.init(in); err != nil {
return err return err
} }
// TODO: Investigate if sending history without decoders are faster. if err := seqs.initialize(br, hist, b.dst); err != nil {
// This would allow the sequences to be decoded async and only have to construct stream history. println("initializing sequences:", err)
// If only recent offsets were not transferred, this would be an obvious win. return err
// Also, if first 3 sequences don't reference recent offsets, all sequences can be decoded. }
return nil
}
func (b *blockDec) decodeSequences(hist *history) error {
if cap(b.sequence) < hist.decoders.nSeqs {
if b.lowMem {
b.sequence = make([]seqVals, 0, hist.decoders.nSeqs)
} else {
b.sequence = make([]seqVals, 0, 0x7F00+0xffff)
}
}
b.sequence = b.sequence[:hist.decoders.nSeqs]
if hist.decoders.nSeqs == 0 {
hist.decoders.seqSize = len(hist.decoders.literals)
return nil
}
hist.decoders.windowSize = hist.windowSize
hist.decoders.prevOffset = hist.recentOffsets
err := hist.decoders.decode(b.sequence)
hist.recentOffsets = hist.decoders.prevOffset
return err
}
func (b *blockDec) executeSequences(hist *history) error {
hbytes := hist.b hbytes := hist.b
if len(hbytes) > hist.windowSize { if len(hbytes) > hist.windowSize {
hbytes = hbytes[len(hbytes)-hist.windowSize:] hbytes = hbytes[len(hbytes)-hist.windowSize:]
@ -678,47 +664,35 @@ func (b *blockDec) decodeCompressed(hist *history) error {
hist.dict.content = nil hist.dict.content = nil
} }
} }
hist.decoders.windowSize = hist.windowSize
if err := seqs.initialize(br, hist, literals, b.dst); err != nil { hist.decoders.out = b.dst[:0]
println("initializing sequences:", err) err := hist.decoders.execute(b.sequence, hbytes)
return err
}
err = seqs.decode(nSeqs, br, hbytes)
if err != nil { if err != nil {
return err return err
} }
if !br.finished() { return b.updateHistory(hist)
return fmt.Errorf("%d extra bits on block, should be 0", br.remain())
} }
err = br.close() func (b *blockDec) updateHistory(hist *history) error {
if err != nil {
printf("Closing sequences: %v, %+v\n", err, *br)
}
if len(b.data) > maxCompressedBlockSize { if len(b.data) > maxCompressedBlockSize {
return fmt.Errorf("compressed block size too large (%d)", len(b.data)) return fmt.Errorf("compressed block size too large (%d)", len(b.data))
} }
// Set output and release references. // Set output and release references.
b.dst = seqs.out b.dst = hist.decoders.out
seqs.out, seqs.literals, seqs.hist = nil, nil, nil hist.recentOffsets = hist.decoders.prevOffset
if !delayedHistory {
// If we don't have delayed history, no need to update.
hist.recentOffsets = seqs.prevOffset
return nil
}
if b.Last { if b.Last {
// if last block we don't care about history. // if last block we don't care about history.
println("Last block, no history returned") println("Last block, no history returned")
hist.b = hist.b[:0] hist.b = hist.b[:0]
return nil return nil
} } else {
hist.append(b.dst) hist.append(b.dst)
hist.recentOffsets = seqs.prevOffset
if debugDecoder { if debugDecoder {
println("Finished block with literals:", len(literals), "and", nSeqs, "sequences.") println("Finished block with ", len(b.sequence), "sequences. Added", len(b.dst), "to history, now length", len(hist.b))
} }
}
hist.decoders.out, hist.decoders.literals = nil, nil
return nil return nil
} }

3
vendor/github.com/klauspost/compress/zstd/bytebuf.go generated vendored
View File

@ -113,6 +113,9 @@ func (r *readerWrapper) readBig(n int, dst []byte) ([]byte, error) {
func (r *readerWrapper) readByte() (byte, error) { func (r *readerWrapper) readByte() (byte, error) {
n2, err := r.r.Read(r.tmp[:1]) n2, err := r.r.Read(r.tmp[:1])
if err != nil { if err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
return 0, err return 0, err
} }
if n2 != 1 { if n2 != 1 {

601
vendor/github.com/klauspost/compress/zstd/decoder.go generated vendored
View File

@ -5,9 +5,13 @@
package zstd package zstd
import ( import (
"errors" "bytes"
"context"
"encoding/binary"
"io" "io"
"sync" "sync"
"github.com/klauspost/compress/zstd/internal/xxhash"
) )
// Decoder provides decoding of zstandard streams. // Decoder provides decoding of zstandard streams.
@ -22,12 +26,19 @@ type Decoder struct {
// Unreferenced decoders, ready for use. // Unreferenced decoders, ready for use.
decoders chan *blockDec decoders chan *blockDec
// Streams ready to be decoded.
stream chan decodeStream
// Current read position used for Reader functionality. // Current read position used for Reader functionality.
current decoderState current decoderState
// sync stream decoding
syncStream struct {
decodedFrame uint64
br readerWrapper
enabled bool
inFrame bool
}
frame *frameDec
// Custom dictionaries. // Custom dictionaries.
// Always uses copies. // Always uses copies.
dicts map[uint32]dict dicts map[uint32]dict
@ -46,7 +57,10 @@ type decoderState struct {
output chan decodeOutput output chan decodeOutput
// cancel remaining output. // cancel remaining output.
cancel chan struct{} cancel context.CancelFunc
// crc of current frame
crc *xxhash.Digest
flushed bool flushed bool
} }
@ -81,7 +95,7 @@ func NewReader(r io.Reader, opts ...DOption) (*Decoder, error) {
return nil, err return nil, err
} }
} }
d.current.output = make(chan decodeOutput, d.o.concurrent) d.current.crc = xxhash.New()
d.current.flushed = true d.current.flushed = true
if r == nil { if r == nil {
@ -130,7 +144,7 @@ func (d *Decoder) Read(p []byte) (int, error) {
break break
} }
if !d.nextBlock(n == 0) { if !d.nextBlock(n == 0) {
return n, nil return n, d.current.err
} }
} }
} }
@ -162,6 +176,7 @@ func (d *Decoder) Reset(r io.Reader) error {
d.drainOutput() d.drainOutput()
d.syncStream.br.r = nil
if r == nil { if r == nil {
d.current.err = ErrDecoderNilInput d.current.err = ErrDecoderNilInput
if len(d.current.b) > 0 { if len(d.current.b) > 0 {
@ -195,33 +210,39 @@ func (d *Decoder) Reset(r io.Reader) error {
} }
return nil return nil
} }
if d.stream == nil {
d.stream = make(chan decodeStream, 1)
d.streamWg.Add(1)
go d.startStreamDecoder(d.stream)
}
// Remove current block. // Remove current block.
d.stashDecoder()
d.current.decodeOutput = decodeOutput{} d.current.decodeOutput = decodeOutput{}
d.current.err = nil d.current.err = nil
d.current.cancel = make(chan struct{})
d.current.flushed = false d.current.flushed = false
d.current.d = nil d.current.d = nil
d.stream <- decodeStream{ // Ensure no-one else is still running...
r: r, d.streamWg.Wait()
output: d.current.output, if d.frame == nil {
cancel: d.current.cancel, d.frame = newFrameDec(d.o)
} }
if d.o.concurrent == 1 {
return d.startSyncDecoder(r)
}
d.current.output = make(chan decodeOutput, d.o.concurrent)
ctx, cancel := context.WithCancel(context.Background())
d.current.cancel = cancel
d.streamWg.Add(1)
go d.startStreamDecoder(ctx, r, d.current.output)
return nil return nil
} }
// drainOutput will drain the output until errEndOfStream is sent. // drainOutput will drain the output until errEndOfStream is sent.
func (d *Decoder) drainOutput() { func (d *Decoder) drainOutput() {
if d.current.cancel != nil { if d.current.cancel != nil {
if debugDecoder {
println("cancelling current") println("cancelling current")
close(d.current.cancel) }
d.current.cancel()
d.current.cancel = nil d.current.cancel = nil
} }
if d.current.d != nil { if d.current.d != nil {
@ -243,12 +264,9 @@ func (d *Decoder) drainOutput() {
} }
d.decoders <- v.d d.decoders <- v.d
} }
if v.err == errEndOfStream { }
println("current flushed") d.current.output = nil
d.current.flushed = true d.current.flushed = true
return
}
}
} }
// WriteTo writes data to w until there's no more data to write or when an error occurs. // WriteTo writes data to w until there's no more data to write or when an error occurs.
@ -287,7 +305,7 @@ func (d *Decoder) WriteTo(w io.Writer) (int64, error) {
// DecodeAll can be used concurrently. // DecodeAll can be used concurrently.
// The Decoder concurrency limits will be respected. // The Decoder concurrency limits will be respected.
func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) { func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) {
if d.current.err == ErrDecoderClosed { if d.decoders == nil {
return dst, ErrDecoderClosed return dst, ErrDecoderClosed
} }
@ -300,6 +318,9 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) {
} }
frame.rawInput = nil frame.rawInput = nil
frame.bBuf = nil frame.bBuf = nil
if frame.history.decoders.br != nil {
frame.history.decoders.br.in = nil
}
d.decoders <- block d.decoders <- block
}() }()
frame.bBuf = input frame.bBuf = input
@ -307,27 +328,31 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) {
for { for {
frame.history.reset() frame.history.reset()
err := frame.reset(&frame.bBuf) err := frame.reset(&frame.bBuf)
if err != nil {
if err == io.EOF { if err == io.EOF {
if debugDecoder { if debugDecoder {
println("frame reset return EOF") println("frame reset return EOF")
} }
return dst, nil return dst, nil
} }
return dst, err
}
if frame.DictionaryID != nil { if frame.DictionaryID != nil {
dict, ok := d.dicts[*frame.DictionaryID] dict, ok := d.dicts[*frame.DictionaryID]
if !ok { if !ok {
return nil, ErrUnknownDictionary return nil, ErrUnknownDictionary
} }
if debugDecoder {
println("setting dict", frame.DictionaryID)
}
frame.history.setDict(&dict) frame.history.setDict(&dict)
} }
if err != nil {
return dst, err if frame.FrameContentSize != fcsUnknown && frame.FrameContentSize > d.o.maxDecodedSize-uint64(len(dst)) {
}
if frame.FrameContentSize > d.o.maxDecodedSize-uint64(len(dst)) {
return dst, ErrDecoderSizeExceeded return dst, ErrDecoderSizeExceeded
} }
if frame.FrameContentSize > 0 && frame.FrameContentSize < 1<<30 { if frame.FrameContentSize < 1<<30 {
// Never preallocate moe than 1 GB up front. // Never preallocate more than 1 GB up front.
if cap(dst)-len(dst) < int(frame.FrameContentSize) { if cap(dst)-len(dst) < int(frame.FrameContentSize) {
dst2 := make([]byte, len(dst), len(dst)+int(frame.FrameContentSize)) dst2 := make([]byte, len(dst), len(dst)+int(frame.FrameContentSize))
copy(dst2, dst) copy(dst2, dst)
@ -368,6 +393,161 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) {
// If non-blocking mode is used the returned boolean will be false // If non-blocking mode is used the returned boolean will be false
// if no data was available without blocking. // if no data was available without blocking.
func (d *Decoder) nextBlock(blocking bool) (ok bool) { func (d *Decoder) nextBlock(blocking bool) (ok bool) {
if d.current.err != nil {
// Keep error state.
return false
}
d.current.b = d.current.b[:0]
// SYNC:
if d.syncStream.enabled {
if !blocking {
return false
}
ok = d.nextBlockSync()
if !ok {
d.stashDecoder()
}
return ok
}
//ASYNC:
d.stashDecoder()
if blocking {
d.current.decodeOutput, ok = <-d.current.output
} else {
select {
case d.current.decodeOutput, ok = <-d.current.output:
default:
return false
}
}
if !ok {
// This should not happen, so signal error state...
d.current.err = io.ErrUnexpectedEOF
return false
}
next := d.current.decodeOutput
if next.d != nil && next.d.async.newHist != nil {
d.current.crc.Reset()
}
if debugDecoder {
var tmp [4]byte
binary.LittleEndian.PutUint32(tmp[:], uint32(xxhash.Sum64(next.b)))
println("got", len(d.current.b), "bytes, error:", d.current.err, "data crc:", tmp)
}
if len(next.b) > 0 {
n, err := d.current.crc.Write(next.b)
if err == nil {
if n != len(next.b) {
d.current.err = io.ErrShortWrite
}
}
}
if next.err == nil && next.d != nil && len(next.d.checkCRC) != 0 {
got := d.current.crc.Sum64()
var tmp [4]byte
binary.LittleEndian.PutUint32(tmp[:], uint32(got))
if !bytes.Equal(tmp[:], next.d.checkCRC) && !ignoreCRC {
if debugDecoder {
println("CRC Check Failed:", tmp[:], " (got) !=", next.d.checkCRC, "(on stream)")
}
d.current.err = ErrCRCMismatch
} else {
if debugDecoder {
println("CRC ok", tmp[:])
}
}
}
return true
}
func (d *Decoder) nextBlockSync() (ok bool) {
if d.current.d == nil {
d.current.d = <-d.decoders
}
for len(d.current.b) == 0 {
if !d.syncStream.inFrame {
d.frame.history.reset()
d.current.err = d.frame.reset(&d.syncStream.br)
if d.current.err != nil {
return false
}
if d.frame.DictionaryID != nil {
dict, ok := d.dicts[*d.frame.DictionaryID]
if !ok {
d.current.err = ErrUnknownDictionary
return false
} else {
d.frame.history.setDict(&dict)
}
}
if d.frame.WindowSize > d.o.maxDecodedSize || d.frame.WindowSize > d.o.maxWindowSize {
d.current.err = ErrDecoderSizeExceeded
return false
}
d.syncStream.decodedFrame = 0
d.syncStream.inFrame = true
}
d.current.err = d.frame.next(d.current.d)
if d.current.err != nil {
return false
}
d.frame.history.ensureBlock()
if debugDecoder {
println("History trimmed:", len(d.frame.history.b), "decoded already:", d.syncStream.decodedFrame)
}
histBefore := len(d.frame.history.b)
d.current.err = d.current.d.decodeBuf(&d.frame.history)
if d.current.err != nil {
println("error after:", d.current.err)
return false
}
d.current.b = d.frame.history.b[histBefore:]
if debugDecoder {
println("history after:", len(d.frame.history.b))
}
// Check frame size (before CRC)
d.syncStream.decodedFrame += uint64(len(d.current.b))
if d.syncStream.decodedFrame > d.frame.FrameContentSize {
if debugDecoder {
printf("DecodedFrame (%d) > FrameContentSize (%d)\n", d.syncStream.decodedFrame, d.frame.FrameContentSize)
}
d.current.err = ErrFrameSizeExceeded
return false
}
// Check FCS
if d.current.d.Last && d.frame.FrameContentSize != fcsUnknown && d.syncStream.decodedFrame != d.frame.FrameContentSize {
if debugDecoder {
printf("DecodedFrame (%d) != FrameContentSize (%d)\n", d.syncStream.decodedFrame, d.frame.FrameContentSize)
}
d.current.err = ErrFrameSizeMismatch
return false
}
// Update/Check CRC
if d.frame.HasCheckSum {
d.frame.crc.Write(d.current.b)
if d.current.d.Last {
d.current.err = d.frame.checkCRC()
if d.current.err != nil {
println("CRC error:", d.current.err)
return false
}
}
}
d.syncStream.inFrame = !d.current.d.Last
}
return true
}
func (d *Decoder) stashDecoder() {
if d.current.d != nil { if d.current.d != nil {
if debugDecoder { if debugDecoder {
printf("re-adding current decoder %p", d.current.d) printf("re-adding current decoder %p", d.current.d)
@ -375,24 +555,6 @@ func (d *Decoder) nextBlock(blocking bool) (ok bool) {
d.decoders <- d.current.d d.decoders <- d.current.d
d.current.d = nil d.current.d = nil
} }
if d.current.err != nil {
// Keep error state.
return blocking
}
if blocking {
d.current.decodeOutput = <-d.current.output
} else {
select {
case d.current.decodeOutput = <-d.current.output:
default:
return false
}
}
if debugDecoder {
println("got", len(d.current.b), "bytes, error:", d.current.err)
}
return true
} }
// Close will release all resources. // Close will release all resources.
@ -402,10 +564,10 @@ func (d *Decoder) Close() {
return return
} }
d.drainOutput() d.drainOutput()
if d.stream != nil { if d.current.cancel != nil {
close(d.stream) d.current.cancel()
d.streamWg.Wait() d.streamWg.Wait()
d.stream = nil d.current.cancel = nil
} }
if d.decoders != nil { if d.decoders != nil {
close(d.decoders) close(d.decoders)
@ -456,39 +618,230 @@ type decodeOutput struct {
err error err error
} }
type decodeStream struct { func (d *Decoder) startSyncDecoder(r io.Reader) error {
r io.Reader d.frame.history.reset()
d.syncStream.br = readerWrapper{r: r}
// Blocks ready to be written to output. d.syncStream.inFrame = false
output chan decodeOutput d.syncStream.enabled = true
d.syncStream.decodedFrame = 0
// cancel reading from the input return nil
cancel chan struct{}
} }
// errEndOfStream indicates that everything from the stream was read.
var errEndOfStream = errors.New("end-of-stream")
// Create Decoder: // Create Decoder:
// Spawn n block decoders. These accept tasks to decode a block. // ASYNC:
// Create goroutine that handles stream processing, this will send history to decoders as they are available. // Spawn 4 go routines.
// Decoders update the history as they decode. // 0: Read frames and decode blocks.
// When a block is returned: // 1: Decode block and literals. Receives hufftree and seqdecs, returns seqdecs and huff tree.
// a) history is sent to the next decoder, // 2: Wait for recentOffsets if needed. Decode sequences, send recentOffsets.
// b) content written to CRC. // 3: Wait for stream history, execute sequences, send stream history.
// c) return data to WRITER. func (d *Decoder) startStreamDecoder(ctx context.Context, r io.Reader, output chan decodeOutput) {
// d) wait for next block to return data.
// Once WRITTEN, the decoders reused by the writer frame decoder for re-use.
func (d *Decoder) startStreamDecoder(inStream chan decodeStream) {
defer d.streamWg.Done() defer d.streamWg.Done()
frame := newFrameDec(d.o) br := readerWrapper{r: r}
for stream := range inStream {
if debugDecoder { var seqPrepare = make(chan *blockDec, d.o.concurrent)
println("got new stream") var seqDecode = make(chan *blockDec, d.o.concurrent)
var seqExecute = make(chan *blockDec, d.o.concurrent)
// Async 1: Prepare blocks...
go func() {
var hist history
var hasErr bool
for block := range seqPrepare {
if hasErr {
if block != nil {
seqDecode <- block
} }
br := readerWrapper{r: stream.r} continue
}
if block.async.newHist != nil {
if debugDecoder {
println("Async 1: new history")
}
hist.reset()
if block.async.newHist.dict != nil {
hist.setDict(block.async.newHist.dict)
}
}
if block.err != nil || block.Type != blockTypeCompressed {
hasErr = block.err != nil
seqDecode <- block
continue
}
remain, err := block.decodeLiterals(block.data, &hist)
block.err = err
hasErr = block.err != nil
if err == nil {
block.async.literals = hist.decoders.literals
block.async.seqData = remain
} else if debugDecoder {
println("decodeLiterals error:", err)
}
seqDecode <- block
}
close(seqDecode)
}()
// Async 2: Decode sequences...
go func() {
var hist history
var hasErr bool
for block := range seqDecode {
if hasErr {
if block != nil {
seqExecute <- block
}
continue
}
if block.async.newHist != nil {
if debugDecoder {
println("Async 2: new history, recent:", block.async.newHist.recentOffsets)
}
hist.decoders = block.async.newHist.decoders
hist.recentOffsets = block.async.newHist.recentOffsets
hist.windowSize = block.async.newHist.windowSize
if block.async.newHist.dict != nil {
hist.setDict(block.async.newHist.dict)
}
}
if block.err != nil || block.Type != blockTypeCompressed {
hasErr = block.err != nil
seqExecute <- block
continue
}
hist.decoders.literals = block.async.literals
block.err = block.prepareSequences(block.async.seqData, &hist)
if debugDecoder && block.err != nil {
println("prepareSequences returned:", block.err)
}
hasErr = block.err != nil
if block.err == nil {
block.err = block.decodeSequences(&hist)
if debugDecoder && block.err != nil {
println("decodeSequences returned:", block.err)
}
hasErr = block.err != nil
// block.async.sequence = hist.decoders.seq[:hist.decoders.nSeqs]
block.async.seqSize = hist.decoders.seqSize
}
seqExecute <- block
}
close(seqExecute)
}()
var wg sync.WaitGroup
wg.Add(1)
// Async 3: Execute sequences...
frameHistCache := d.frame.history.b
go func() {
var hist history
var decodedFrame uint64
var fcs uint64
var hasErr bool
for block := range seqExecute {
out := decodeOutput{err: block.err, d: block}
if block.err != nil || hasErr {
hasErr = true
output <- out
continue
}
if block.async.newHist != nil {
if debugDecoder {
println("Async 3: new history")
}
hist.windowSize = block.async.newHist.windowSize
hist.allocFrameBuffer = block.async.newHist.allocFrameBuffer
if block.async.newHist.dict != nil {
hist.setDict(block.async.newHist.dict)
}
if cap(hist.b) < hist.allocFrameBuffer {
if cap(frameHistCache) >= hist.allocFrameBuffer {
hist.b = frameHistCache
} else {
hist.b = make([]byte, 0, hist.allocFrameBuffer)
println("Alloc history sized", hist.allocFrameBuffer)
}
}
hist.b = hist.b[:0]
fcs = block.async.fcs
decodedFrame = 0
}
do := decodeOutput{err: block.err, d: block}
switch block.Type {
case blockTypeRLE:
if debugDecoder {
println("add rle block length:", block.RLESize)
}
if cap(block.dst) < int(block.RLESize) {
if block.lowMem {
block.dst = make([]byte, block.RLESize)
} else {
block.dst = make([]byte, maxBlockSize)
}
}
block.dst = block.dst[:block.RLESize]
v := block.data[0]
for i := range block.dst {
block.dst[i] = v
}
hist.append(block.dst)
do.b = block.dst
case blockTypeRaw:
if debugDecoder {
println("add raw block length:", len(block.data))
}
hist.append(block.data)
do.b = block.data
case blockTypeCompressed:
if debugDecoder {
println("execute with history length:", len(hist.b), "window:", hist.windowSize)
}
hist.decoders.seqSize = block.async.seqSize
hist.decoders.literals = block.async.literals
do.err = block.executeSequences(&hist)
hasErr = do.err != nil
if debugDecoder && hasErr {
println("executeSequences returned:", do.err)
}
do.b = block.dst
}
if !hasErr {
decodedFrame += uint64(len(do.b))
if decodedFrame > fcs {
println("fcs exceeded", block.Last, fcs, decodedFrame)
do.err = ErrFrameSizeExceeded
hasErr = true
} else if block.Last && fcs != fcsUnknown && decodedFrame != fcs {
do.err = ErrFrameSizeMismatch
hasErr = true
} else {
if debugDecoder {
println("fcs ok", block.Last, fcs, decodedFrame)
}
}
}
output <- do
}
close(output)
frameHistCache = hist.b
wg.Done()
if debugDecoder {
println("decoder goroutines finished")
}
}()
decodeStream: decodeStream:
for { for {
frame := d.frame
if debugDecoder {
println("New frame...")
}
var historySent bool
frame.history.reset() frame.history.reset()
err := frame.reset(&br) err := frame.reset(&br)
if debugDecoder && err != nil { if debugDecoder && err != nil {
@ -502,54 +855,70 @@ func (d *Decoder) startStreamDecoder(inStream chan decodeStream) {
frame.history.setDict(&dict) frame.history.setDict(&dict)
} }
} }
if err == nil && d.frame.WindowSize > d.o.maxWindowSize {
err = ErrDecoderSizeExceeded
}
if err != nil { if err != nil {
stream.output <- decodeOutput{ select {
err: err, case <-ctx.Done():
case dec := <-d.decoders:
dec.sendErr(err)
seqPrepare <- dec
} }
break break decodeStream
}
if debugDecoder {
println("starting frame decoder")
} }
// This goroutine will forward history between frames.
frame.frameDone.Add(1)
frame.initAsync()
go frame.startDecoder(stream.output)
decodeFrame:
// Go through all blocks of the frame. // Go through all blocks of the frame.
for { for {
dec := <-d.decoders var dec *blockDec
select { select {
case <-stream.cancel: case <-ctx.Done():
if !frame.sendErr(dec, io.EOF) {
// To not let the decoder dangle, send it back.
stream.output <- decodeOutput{d: dec}
}
break decodeStream break decodeStream
default: case dec = <-d.decoders:
// Once we have a decoder, we MUST return it.
} }
err := frame.next(dec) err := frame.next(dec)
switch err { if !historySent {
case io.EOF: h := frame.history
// End of current frame, no error if debugDecoder {
println("EOF on next block") println("Alloc History:", h.allocFrameBuffer)
break decodeFrame }
case nil: dec.async.newHist = &h
continue dec.async.fcs = frame.FrameContentSize
default: historySent = true
println("block decoder returned", err) } else {
dec.async.newHist = nil
}
if debugDecoder && err != nil {
println("next block returned error:", err)
}
dec.err = err
dec.checkCRC = nil
if dec.Last && frame.HasCheckSum && err == nil {
crc, err := frame.rawInput.readSmall(4)
if err != nil {
println("CRC missing?", err)
dec.err = err
}
var tmp [4]byte
copy(tmp[:], crc)
dec.checkCRC = tmp[:]
if debugDecoder {
println("found crc to check:", dec.checkCRC)
}
}
err = dec.err
last := dec.Last
seqPrepare <- dec
if err != nil {
break decodeStream break decodeStream
} }
} if last {
// All blocks have started decoding, check if there are more frames. break
println("waiting for done")
frame.frameDone.Wait()
println("done waiting...")
}
frame.frameDone.Wait()
println("Sending EOS")
stream.output <- decodeOutput{err: errEndOfStream}
} }
} }
}
close(seqPrepare)
wg.Wait()
d.frame.history.b = frameHistCache
}

22
vendor/github.com/klauspost/compress/zstd/decoder_options.go generated vendored
View File

@ -28,6 +28,9 @@ func (o *decoderOptions) setDefault() {
concurrent: runtime.GOMAXPROCS(0), concurrent: runtime.GOMAXPROCS(0),
maxWindowSize: MaxWindowSize, maxWindowSize: MaxWindowSize,
} }
if o.concurrent > 4 {
o.concurrent = 4
}
o.maxDecodedSize = 1 << 63 o.maxDecodedSize = 1 << 63
} }
@ -37,16 +40,25 @@ func WithDecoderLowmem(b bool) DOption {
return func(o *decoderOptions) error { o.lowMem = b; return nil } return func(o *decoderOptions) error { o.lowMem = b; return nil }
} }
// WithDecoderConcurrency will set the concurrency, // WithDecoderConcurrency sets the number of created decoders.
// meaning the maximum number of decoders to run concurrently. // When decoding block with DecodeAll, this will limit the number
// The value supplied must be at least 1. // of possible concurrently running decodes.
// By default this will be set to GOMAXPROCS. // When decoding streams, this will limit the number of
// inflight blocks.
// When decoding streams and setting maximum to 1,
// no async decoding will be done.
// When a value of 0 is provided GOMAXPROCS will be used.
// By default this will be set to 4 or GOMAXPROCS, whatever is lower.
func WithDecoderConcurrency(n int) DOption { func WithDecoderConcurrency(n int) DOption {
return func(o *decoderOptions) error { return func(o *decoderOptions) error {
if n <= 0 { if n < 0 {
return errors.New("concurrency must be at least 1") return errors.New("concurrency must be at least 1")
} }
if n == 0 {
o.concurrent = runtime.GOMAXPROCS(0)
} else {
o.concurrent = n o.concurrent = n
}
return nil return nil
} }
} }

4
vendor/github.com/klauspost/compress/zstd/enc_fast.go generated vendored
View File

@ -85,7 +85,7 @@ func (e *fastEncoder) Encode(blk *blockEnc, src []byte) {
// TEMPLATE // TEMPLATE
const hashLog = tableBits const hashLog = tableBits
// seems global, but would be nice to tweak. // seems global, but would be nice to tweak.
const kSearchStrength = 7 const kSearchStrength = 6
// nextEmit is where in src the next emitLiteral should start from. // nextEmit is where in src the next emitLiteral should start from.
nextEmit := s nextEmit := s
@ -334,7 +334,7 @@ func (e *fastEncoder) EncodeNoHist(blk *blockEnc, src []byte) {
// TEMPLATE // TEMPLATE
const hashLog = tableBits const hashLog = tableBits
// seems global, but would be nice to tweak. // seems global, but would be nice to tweak.
const kSearchStrength = 8 const kSearchStrength = 6
// nextEmit is where in src the next emitLiteral should start from. // nextEmit is where in src the next emitLiteral should start from.
nextEmit := s nextEmit := s

52
vendor/github.com/klauspost/compress/zstd/encoder.go generated vendored
View File

@ -98,23 +98,25 @@ func (e *Encoder) Reset(w io.Writer) {
if cap(s.filling) == 0 { if cap(s.filling) == 0 {
s.filling = make([]byte, 0, e.o.blockSize) s.filling = make([]byte, 0, e.o.blockSize)
} }
if e.o.concurrent > 1 {
if cap(s.current) == 0 { if cap(s.current) == 0 {
s.current = make([]byte, 0, e.o.blockSize) s.current = make([]byte, 0, e.o.blockSize)
} }
if cap(s.previous) == 0 { if cap(s.previous) == 0 {
s.previous = make([]byte, 0, e.o.blockSize) s.previous = make([]byte, 0, e.o.blockSize)
} }
if s.encoder == nil { s.current = s.current[:0]
s.encoder = e.o.encoder() s.previous = s.previous[:0]
}
if s.writing == nil { if s.writing == nil {
s.writing = &blockEnc{lowMem: e.o.lowMem} s.writing = &blockEnc{lowMem: e.o.lowMem}
s.writing.init() s.writing.init()
} }
s.writing.initNewEncode() s.writing.initNewEncode()
}
if s.encoder == nil {
s.encoder = e.o.encoder()
}
s.filling = s.filling[:0] s.filling = s.filling[:0]
s.current = s.current[:0]
s.previous = s.previous[:0]
s.encoder.Reset(e.o.dict, false) s.encoder.Reset(e.o.dict, false)
s.headerWritten = false s.headerWritten = false
s.eofWritten = false s.eofWritten = false
@ -258,6 +260,46 @@ func (e *Encoder) nextBlock(final bool) error {
return s.err return s.err
} }
// SYNC:
if e.o.concurrent == 1 {
src := s.filling
s.nInput += int64(len(s.filling))
if debugEncoder {
println("Adding sync block,", len(src), "bytes, final:", final)
}
enc := s.encoder
blk := enc.Block()
blk.reset(nil)
enc.Encode(blk, src)
blk.last = final
if final {
s.eofWritten = true
}
err := errIncompressible
// If we got the exact same number of literals as input,
// assume the literals cannot be compressed.
if len(src) != len(blk.literals) || len(src) != e.o.blockSize {
err = blk.encode(src, e.o.noEntropy, !e.o.allLitEntropy)
}
switch err {
case errIncompressible:
if debugEncoder {
println("Storing incompressible block as raw")
}
blk.encodeRaw(src)
// In fast mode, we do not transfer offsets, so we don't have to deal with changing the.
case nil:
default:
s.err = err
return err
}
_, s.err = s.w.Write(blk.output)
s.nWritten += int64(len(blk.output))
s.filling = s.filling[:0]
return s.err
}
// Move blocks forward. // Move blocks forward.
s.filling, s.current, s.previous = s.previous[:0], s.filling, s.current s.filling, s.current, s.previous = s.previous[:0], s.filling, s.current
s.nInput += int64(len(s.current)) s.nInput += int64(len(s.current))

1
vendor/github.com/klauspost/compress/zstd/encoder_options.go generated vendored
View File

@ -76,6 +76,7 @@ func WithEncoderCRC(b bool) EOption {
// WithEncoderConcurrency will set the concurrency, // WithEncoderConcurrency will set the concurrency,
// meaning the maximum number of encoders to run concurrently. // meaning the maximum number of encoders to run concurrently.
// The value supplied must be at least 1. // The value supplied must be at least 1.
// For streams, setting a value of 1 will disable async compression.
// By default this will be set to GOMAXPROCS. // By default this will be set to GOMAXPROCS.
func WithEncoderConcurrency(n int) EOption { func WithEncoderConcurrency(n int) EOption {
return func(o *encoderOptions) error { return func(o *encoderOptions) error {

201
vendor/github.com/klauspost/compress/zstd/framedec.go generated vendored
View File

@ -8,23 +8,17 @@ import (
"bytes" "bytes"
"encoding/hex" "encoding/hex"
"errors" "errors"
"hash"
"io" "io"
"sync"
"github.com/klauspost/compress/zstd/internal/xxhash" "github.com/klauspost/compress/zstd/internal/xxhash"
) )
type frameDec struct { type frameDec struct {
o decoderOptions o decoderOptions
crc hash.Hash64 crc *xxhash.Digest
offset int64
WindowSize uint64 WindowSize uint64
// In order queue of blocks being decoded.
decoding chan *blockDec
// Frame history passed between blocks // Frame history passed between blocks
history history history history
@ -34,15 +28,10 @@ type frameDec struct {
bBuf byteBuf bBuf byteBuf
FrameContentSize uint64 FrameContentSize uint64
frameDone sync.WaitGroup
DictionaryID *uint32 DictionaryID *uint32
HasCheckSum bool HasCheckSum bool
SingleSegment bool SingleSegment bool
// asyncRunning indicates whether the async routine processes input on 'decoding'.
asyncRunningMu sync.Mutex
asyncRunning bool
} }
const ( const (
@ -208,7 +197,7 @@ func (d *frameDec) reset(br byteBuffer) error {
default: default:
fcsSize = 1 << v fcsSize = 1 << v
} }
d.FrameContentSize = 0 d.FrameContentSize = fcsUnknown
if fcsSize > 0 { if fcsSize > 0 {
b, err := br.readSmall(fcsSize) b, err := br.readSmall(fcsSize)
if err != nil { if err != nil {
@ -229,9 +218,10 @@ func (d *frameDec) reset(br byteBuffer) error {
d.FrameContentSize = uint64(d1) | (uint64(d2) << 32) d.FrameContentSize = uint64(d1) | (uint64(d2) << 32)
} }
if debugDecoder { if debugDecoder {
println("field size bits:", v, "fcsSize:", fcsSize, "FrameContentSize:", d.FrameContentSize, hex.EncodeToString(b[:fcsSize]), "singleseg:", d.SingleSegment, "window:", d.WindowSize) println("Read FCS:", d.FrameContentSize)
} }
} }
// Move this to shared. // Move this to shared.
d.HasCheckSum = fhd&(1<<2) != 0 d.HasCheckSum = fhd&(1<<2) != 0
if d.HasCheckSum { if d.HasCheckSum {
@ -264,10 +254,16 @@ func (d *frameDec) reset(br byteBuffer) error {
} }
d.history.windowSize = int(d.WindowSize) d.history.windowSize = int(d.WindowSize)
if d.o.lowMem && d.history.windowSize < maxBlockSize { if d.o.lowMem && d.history.windowSize < maxBlockSize {
d.history.maxSize = d.history.windowSize * 2 d.history.allocFrameBuffer = d.history.windowSize * 2
// TODO: Maybe use FrameContent size
} else { } else {
d.history.maxSize = d.history.windowSize + maxBlockSize d.history.allocFrameBuffer = d.history.windowSize + maxBlockSize
} }
if debugDecoder {
println("Frame: Dict:", d.DictionaryID, "FrameContentSize:", d.FrameContentSize, "singleseg:", d.SingleSegment, "window:", d.WindowSize, "crc:", d.HasCheckSum)
}
// history contains input - maybe we do something // history contains input - maybe we do something
d.rawInput = br d.rawInput = br
return nil return nil
@ -276,48 +272,17 @@ func (d *frameDec) reset(br byteBuffer) error {
// next will start decoding the next block from stream. // next will start decoding the next block from stream.
func (d *frameDec) next(block *blockDec) error { func (d *frameDec) next(block *blockDec) error {
if debugDecoder { if debugDecoder {
printf("decoding new block %p:%p", block, block.data) println("decoding new block")
} }
err := block.reset(d.rawInput, d.WindowSize) err := block.reset(d.rawInput, d.WindowSize)
if err != nil { if err != nil {
println("block error:", err) println("block error:", err)
// Signal the frame decoder we have a problem. // Signal the frame decoder we have a problem.
d.sendErr(block, err) block.sendErr(err)
return err return err
} }
block.input <- struct{}{}
if debugDecoder {
println("next block:", block)
}
d.asyncRunningMu.Lock()
defer d.asyncRunningMu.Unlock()
if !d.asyncRunning {
return nil return nil
} }
if block.Last {
// We indicate the frame is done by sending io.EOF
d.decoding <- block
return io.EOF
}
d.decoding <- block
return nil
}
// sendEOF will queue an error block on the frame.
// This will cause the frame decoder to return when it encounters the block.
// Returns true if the decoder was added.
func (d *frameDec) sendErr(block *blockDec, err error) bool {
d.asyncRunningMu.Lock()
defer d.asyncRunningMu.Unlock()
if !d.asyncRunning {
return false
}
println("sending error", err.Error())
block.sendErr(err)
d.decoding <- block
return true
}
// checkCRC will check the checksum if the frame has one. // checkCRC will check the checksum if the frame has one.
// Will return ErrCRCMismatch if crc check failed, otherwise nil. // Will return ErrCRCMismatch if crc check failed, otherwise nil.
@ -340,7 +305,7 @@ func (d *frameDec) checkCRC() error {
return err return err
} }
if !bytes.Equal(tmp[:], want) { if !bytes.Equal(tmp[:], want) && !ignoreCRC {
if debugDecoder { if debugDecoder {
println("CRC Check Failed:", tmp[:], "!=", want) println("CRC Check Failed:", tmp[:], "!=", want)
} }
@ -352,131 +317,13 @@ func (d *frameDec) checkCRC() error {
return nil return nil
} }
func (d *frameDec) initAsync() {
if !d.o.lowMem && !d.SingleSegment {
// set max extra size history to 2MB.
d.history.maxSize = d.history.windowSize + maxBlockSize
}
// re-alloc if more than one extra block size.
if d.o.lowMem && cap(d.history.b) > d.history.maxSize+maxBlockSize {
d.history.b = make([]byte, 0, d.history.maxSize)
}
if cap(d.history.b) < d.history.maxSize {
d.history.b = make([]byte, 0, d.history.maxSize)
}
if cap(d.decoding) < d.o.concurrent {
d.decoding = make(chan *blockDec, d.o.concurrent)
}
if debugDecoder {
h := d.history
printf("history init. len: %d, cap: %d", len(h.b), cap(h.b))
}
d.asyncRunningMu.Lock()
d.asyncRunning = true
d.asyncRunningMu.Unlock()
}
// startDecoder will start decoding blocks and write them to the writer.
// The decoder will stop as soon as an error occurs or at end of frame.
// When the frame has finished decoding the *bufio.Reader
// containing the remaining input will be sent on frameDec.frameDone.
func (d *frameDec) startDecoder(output chan decodeOutput) {
written := int64(0)
defer func() {
d.asyncRunningMu.Lock()
d.asyncRunning = false
d.asyncRunningMu.Unlock()
// Drain the currently decoding.
d.history.error = true
flushdone:
for {
select {
case b := <-d.decoding:
b.history <- &d.history
output <- <-b.result
default:
break flushdone
}
}
println("frame decoder done, signalling done")
d.frameDone.Done()
}()
// Get decoder for first block.
block := <-d.decoding
block.history <- &d.history
for {
var next *blockDec
// Get result
r := <-block.result
if r.err != nil {
println("Result contained error", r.err)
output <- r
return
}
if debugDecoder {
println("got result, from ", d.offset, "to", d.offset+int64(len(r.b)))
d.offset += int64(len(r.b))
}
if !block.Last {
// Send history to next block
select {
case next = <-d.decoding:
if debugDecoder {
println("Sending ", len(d.history.b), "bytes as history")
}
next.history <- &d.history
default:
// Wait until we have sent the block, so
// other decoders can potentially get the decoder.
next = nil
}
}
// Add checksum, async to decoding.
if d.HasCheckSum {
n, err := d.crc.Write(r.b)
if err != nil {
r.err = err
if n != len(r.b) {
r.err = io.ErrShortWrite
}
output <- r
return
}
}
written += int64(len(r.b))
if d.SingleSegment && uint64(written) > d.FrameContentSize {
println("runDecoder: single segment and", uint64(written), ">", d.FrameContentSize)
r.err = ErrFrameSizeExceeded
output <- r
return
}
if block.Last {
r.err = d.checkCRC()
output <- r
return
}
output <- r
if next == nil {
// There was no decoder available, we wait for one now that we have sent to the writer.
if debugDecoder {
println("Sending ", len(d.history.b), " bytes as history")
}
next = <-d.decoding
next.history <- &d.history
}
block = next
}
}
// runDecoder will create a sync decoder that will decode a block of data. // runDecoder will create a sync decoder that will decode a block of data.
func (d *frameDec) runDecoder(dst []byte, dec *blockDec) ([]byte, error) { func (d *frameDec) runDecoder(dst []byte, dec *blockDec) ([]byte, error) {
saved := d.history.b saved := d.history.b
// We use the history for output to avoid copying it. // We use the history for output to avoid copying it.
d.history.b = dst d.history.b = dst
d.history.ignoreBuffer = len(dst)
// Store input length, so we only check new data. // Store input length, so we only check new data.
crcStart := len(dst) crcStart := len(dst)
var err error var err error
@ -489,22 +336,30 @@ func (d *frameDec) runDecoder(dst []byte, dec *blockDec) ([]byte, error) {
println("next block:", dec) println("next block:", dec)
} }
err = dec.decodeBuf(&d.history) err = dec.decodeBuf(&d.history)
if err != nil || dec.Last { if err != nil {
break break
} }
if uint64(len(d.history.b)) > d.o.maxDecodedSize { if uint64(len(d.history.b)) > d.o.maxDecodedSize {
err = ErrDecoderSizeExceeded err = ErrDecoderSizeExceeded
break break
} }
if d.SingleSegment && uint64(len(d.history.b)) > d.o.maxDecodedSize { if uint64(len(d.history.b)-crcStart) > d.FrameContentSize {
println("runDecoder: single segment and", uint64(len(d.history.b)), ">", d.o.maxDecodedSize) println("runDecoder: FrameContentSize exceeded", uint64(len(d.history.b)-crcStart), ">", d.FrameContentSize)
err = ErrFrameSizeExceeded err = ErrFrameSizeExceeded
break break
} }
if dec.Last {
break
}
if debugDecoder {
println("runDecoder: FrameContentSize", uint64(len(d.history.b)-crcStart), "<=", d.FrameContentSize)
}
} }
dst = d.history.b dst = d.history.b
if err == nil { if err == nil {
if d.HasCheckSum { if d.FrameContentSize != fcsUnknown && uint64(len(d.history.b)-crcStart) != d.FrameContentSize {
err = ErrFrameSizeMismatch
} else if d.HasCheckSum {
var n int var n int
n, err = d.crc.Write(dst[crcStart:]) n, err = d.crc.Write(dst[crcStart:])
if err == nil { if err == nil {

11
vendor/github.com/klauspost/compress/zstd/fuzz.go generated vendored Normal file
View File

@ -0,0 +1,11 @@
//go:build ignorecrc
// +build ignorecrc
// Copyright 2019+ Klaus Post. All rights reserved.
// License information can be found in the LICENSE file.
// Based on work by Yann Collet, released under BSD License.
package zstd
// ignoreCRC can be used for fuzz testing to ignore CRC values...
const ignoreCRC = true

11
vendor/github.com/klauspost/compress/zstd/fuzz_none.go generated vendored Normal file
View File

@ -0,0 +1,11 @@
//go:build !ignorecrc
// +build !ignorecrc
// Copyright 2019+ Klaus Post. All rights reserved.
// License information can be found in the LICENSE file.
// Based on work by Yann Collet, released under BSD License.
package zstd
// ignoreCRC can be used for fuzz testing to ignore CRC values...
const ignoreCRC = false

38
vendor/github.com/klauspost/compress/zstd/history.go generated vendored
View File

@ -10,12 +10,22 @@ import (
// history contains the information transferred between blocks. // history contains the information transferred between blocks.
type history struct { type history struct {
b []byte // Literal decompression
huffTree *huff0.Scratch huffTree *huff0.Scratch
recentOffsets [3]int
// Sequence decompression
decoders sequenceDecs decoders sequenceDecs
recentOffsets [3]int
// History buffer...
b []byte
// ignoreBuffer is meant to ignore a number of bytes
// when checking for matches in history
ignoreBuffer int
windowSize int windowSize int
maxSize int allocFrameBuffer int // needed?
error bool error bool
dict *dict dict *dict
} }
@ -24,6 +34,7 @@ type history struct {
// The history must already have been initialized to the desired size. // The history must already have been initialized to the desired size.
func (h *history) reset() { func (h *history) reset() {
h.b = h.b[:0] h.b = h.b[:0]
h.ignoreBuffer = 0
h.error = false h.error = false
h.recentOffsets = [3]int{1, 4, 8} h.recentOffsets = [3]int{1, 4, 8}
if f := h.decoders.litLengths.fse; f != nil && !f.preDefined { if f := h.decoders.litLengths.fse; f != nil && !f.preDefined {
@ -35,7 +46,7 @@ func (h *history) reset() {
if f := h.decoders.matchLengths.fse; f != nil && !f.preDefined { if f := h.decoders.matchLengths.fse; f != nil && !f.preDefined {
fseDecoderPool.Put(f) fseDecoderPool.Put(f)
} }
h.decoders = sequenceDecs{} h.decoders = sequenceDecs{br: h.decoders.br}
if h.huffTree != nil { if h.huffTree != nil {
if h.dict == nil || h.dict.litEnc != h.huffTree { if h.dict == nil || h.dict.litEnc != h.huffTree {
huffDecoderPool.Put(h.huffTree) huffDecoderPool.Put(h.huffTree)
@ -54,6 +65,7 @@ func (h *history) setDict(dict *dict) {
h.decoders.litLengths = dict.llDec h.decoders.litLengths = dict.llDec
h.decoders.offsets = dict.ofDec h.decoders.offsets = dict.ofDec
h.decoders.matchLengths = dict.mlDec h.decoders.matchLengths = dict.mlDec
h.decoders.dict = dict.content
h.recentOffsets = dict.offsets h.recentOffsets = dict.offsets
h.huffTree = dict.litEnc h.huffTree = dict.litEnc
} }
@ -83,6 +95,24 @@ func (h *history) append(b []byte) {
copy(h.b[h.windowSize-len(b):], b) copy(h.b[h.windowSize-len(b):], b)
} }
// ensureBlock will ensure there is space for at least one block...
func (h *history) ensureBlock() {
if cap(h.b) < h.allocFrameBuffer {
h.b = make([]byte, 0, h.allocFrameBuffer)
return
}
avail := cap(h.b) - len(h.b)
if avail >= h.windowSize || avail > maxCompressedBlockSize {
return
}
// Move data down so we only have window size left.
// We know we have less than window size in b at this point.
discard := len(h.b) - h.windowSize
copy(h.b, h.b[discard:])
h.b = h.b[:h.windowSize]
}
// append bytes to history without ever discarding anything. // append bytes to history without ever discarding anything.
func (h *history) appendKeep(b []byte) { func (h *history) appendKeep(b []byte) {
h.b = append(h.b, b...) h.b = append(h.b, b...)

349
vendor/github.com/klauspost/compress/zstd/seqdec.go generated vendored
View File

@ -20,6 +20,10 @@ type seq struct {
llCode, mlCode, ofCode uint8 llCode, mlCode, ofCode uint8
} }
type seqVals struct {
ll, ml, mo int
}
func (s seq) String() string { func (s seq) String() string {
if s.offset <= 3 { if s.offset <= 3 {
if s.offset == 0 { if s.offset == 0 {
@ -61,16 +65,18 @@ type sequenceDecs struct {
offsets sequenceDec offsets sequenceDec
matchLengths sequenceDec matchLengths sequenceDec
prevOffset [3]int prevOffset [3]int
hist []byte
dict []byte dict []byte
literals []byte literals []byte
out []byte out []byte
nSeqs int
br *bitReader
seqSize int
windowSize int windowSize int
maxBits uint8 maxBits uint8
} }
// initialize all 3 decoders from the stream input. // initialize all 3 decoders from the stream input.
func (s *sequenceDecs) initialize(br *bitReader, hist *history, literals, out []byte) error { func (s *sequenceDecs) initialize(br *bitReader, hist *history, out []byte) error {
if err := s.litLengths.init(br); err != nil { if err := s.litLengths.init(br); err != nil {
return errors.New("litLengths:" + err.Error()) return errors.New("litLengths:" + err.Error())
} }
@ -80,8 +86,7 @@ func (s *sequenceDecs) initialize(br *bitReader, hist *history, literals, out []
if err := s.matchLengths.init(br); err != nil { if err := s.matchLengths.init(br); err != nil {
return errors.New("matchLengths:" + err.Error()) return errors.New("matchLengths:" + err.Error())
} }
s.literals = literals s.br = br
s.hist = hist.b
s.prevOffset = hist.recentOffsets s.prevOffset = hist.recentOffsets
s.maxBits = s.litLengths.fse.maxBits + s.offsets.fse.maxBits + s.matchLengths.fse.maxBits s.maxBits = s.litLengths.fse.maxBits + s.offsets.fse.maxBits + s.matchLengths.fse.maxBits
s.windowSize = hist.windowSize s.windowSize = hist.windowSize
@ -94,11 +99,261 @@ func (s *sequenceDecs) initialize(br *bitReader, hist *history, literals, out []
} }
// decode sequences from the stream with the provided history. // decode sequences from the stream with the provided history.
func (s *sequenceDecs) decode(seqs int, br *bitReader, hist []byte) error { func (s *sequenceDecs) decode(seqs []seqVals) error {
br := s.br
// Grab full sizes tables, to avoid bounds checks.
llTable, mlTable, ofTable := s.litLengths.fse.dt[:maxTablesize], s.matchLengths.fse.dt[:maxTablesize], s.offsets.fse.dt[:maxTablesize]
llState, mlState, ofState := s.litLengths.state.state, s.matchLengths.state.state, s.offsets.state.state
s.seqSize = 0
litRemain := len(s.literals)
maxBlockSize := maxCompressedBlockSize
if s.windowSize < maxBlockSize {
maxBlockSize = s.windowSize
}
for i := range seqs {
var ll, mo, ml int
if br.off > 4+((maxOffsetBits+16+16)>>3) {
// inlined function:
// ll, mo, ml = s.nextFast(br, llState, mlState, ofState)
// Final will not read from stream.
var llB, mlB, moB uint8
ll, llB = llState.final()
ml, mlB = mlState.final()
mo, moB = ofState.final()
// extra bits are stored in reverse order.
br.fillFast()
mo += br.getBits(moB)
if s.maxBits > 32 {
br.fillFast()
}
ml += br.getBits(mlB)
ll += br.getBits(llB)
if moB > 1 {
s.prevOffset[2] = s.prevOffset[1]
s.prevOffset[1] = s.prevOffset[0]
s.prevOffset[0] = mo
} else {
// mo = s.adjustOffset(mo, ll, moB)
// Inlined for rather big speedup
if ll == 0 {
// There is an exception though, when current sequence's literals_length = 0.
// In this case, repeated offsets are shifted by one, so an offset_value of 1 means Repeated_Offset2,
// an offset_value of 2 means Repeated_Offset3, and an offset_value of 3 means Repeated_Offset1 - 1_byte.
mo++
}
if mo == 0 {
mo = s.prevOffset[0]
} else {
var temp int
if mo == 3 {
temp = s.prevOffset[0] - 1
} else {
temp = s.prevOffset[mo]
}
if temp == 0 {
// 0 is not valid; input is corrupted; force offset to 1
println("WARNING: temp was 0")
temp = 1
}
if mo != 1 {
s.prevOffset[2] = s.prevOffset[1]
}
s.prevOffset[1] = s.prevOffset[0]
s.prevOffset[0] = temp
mo = temp
}
}
br.fillFast()
} else {
if br.overread() {
if debugDecoder {
printf("reading sequence %d, exceeded available data\n", i)
}
return io.ErrUnexpectedEOF
}
ll, mo, ml = s.next(br, llState, mlState, ofState)
br.fill()
}
if debugSequences {
println("Seq", i, "Litlen:", ll, "mo:", mo, "(abs) ml:", ml)
}
// Evaluate.
// We might be doing this async, so do it early.
if mo == 0 && ml > 0 {
return fmt.Errorf("zero matchoff and matchlen (%d) > 0", ml)
}
if ml > maxMatchLen {
return fmt.Errorf("match len (%d) bigger than max allowed length", ml)
}
s.seqSize += ll + ml
if s.seqSize > maxBlockSize {
return fmt.Errorf("output (%d) bigger than max block size (%d)", s.seqSize, maxBlockSize)
}
litRemain -= ll
if litRemain < 0 {
return fmt.Errorf("unexpected literal count, want %d bytes, but only %d is available", ll, litRemain+ll)
}
seqs[i] = seqVals{
ll: ll,
ml: ml,
mo: mo,
}
if i == len(seqs)-1 {
// This is the last sequence, so we shouldn't update state.
break
}
// Manually inlined, ~ 5-20% faster
// Update all 3 states at once. Approx 20% faster.
nBits := llState.nbBits() + mlState.nbBits() + ofState.nbBits()
if nBits == 0 {
llState = llTable[llState.newState()&maxTableMask]
mlState = mlTable[mlState.newState()&maxTableMask]
ofState = ofTable[ofState.newState()&maxTableMask]
} else {
bits := br.get32BitsFast(nBits)
lowBits := uint16(bits >> ((ofState.nbBits() + mlState.nbBits()) & 31))
llState = llTable[(llState.newState()+lowBits)&maxTableMask]
lowBits = uint16(bits >> (ofState.nbBits() & 31))
lowBits &= bitMask[mlState.nbBits()&15]
mlState = mlTable[(mlState.newState()+lowBits)&maxTableMask]
lowBits = uint16(bits) & bitMask[ofState.nbBits()&15]
ofState = ofTable[(ofState.newState()+lowBits)&maxTableMask]
}
}
s.seqSize += litRemain
if s.seqSize > maxBlockSize {
return fmt.Errorf("output (%d) bigger than max block size (%d)", s.seqSize, maxBlockSize)
}
err := br.close()
if err != nil {
printf("Closing sequences: %v, %+v\n", err, *br)
}
return err
}
// execute will execute the decoded sequence with the provided history.
// The sequence must be evaluated before being sent.
func (s *sequenceDecs) execute(seqs []seqVals, hist []byte) error {
// Ensure we have enough output size...
if len(s.out)+s.seqSize > cap(s.out) {
addBytes := s.seqSize + len(s.out)
s.out = append(s.out, make([]byte, addBytes)...)
s.out = s.out[:len(s.out)-addBytes]
}
if debugDecoder {
printf("Execute %d seqs with hist %d, dict %d, literals: %d into %d bytes\n", len(seqs), len(hist), len(s.dict), len(s.literals), s.seqSize)
}
var t = len(s.out)
out := s.out[:t+s.seqSize]
for _, seq := range seqs {
// Add literals
copy(out[t:], s.literals[:seq.ll])
t += seq.ll
s.literals = s.literals[seq.ll:]
// Copy from dictionary...
if seq.mo > t+len(hist) || seq.mo > s.windowSize {
if len(s.dict) == 0 {
return fmt.Errorf("match offset (%d) bigger than current history (%d)", seq.mo, t+len(hist))
}
// we may be in dictionary.
dictO := len(s.dict) - (seq.mo - (t + len(hist)))
if dictO < 0 || dictO >= len(s.dict) {
return fmt.Errorf("match offset (%d) bigger than current history+dict (%d)", seq.mo, t+len(hist)+len(s.dict))
}
end := dictO + seq.ml
if end > len(s.dict) {
n := len(s.dict) - dictO
copy(out[t:], s.dict[dictO:])
t += n
seq.ml -= n
} else {
copy(out[t:], s.dict[dictO:end])
t += end - dictO
continue
}
}
// Copy from history.
if v := seq.mo - t; v > 0 {
// v is the start position in history from end.
start := len(hist) - v
if seq.ml > v {
// Some goes into current block.
// Copy remainder of history
copy(out[t:], hist[start:])
t += v
seq.ml -= v
} else {
copy(out[t:], hist[start:start+seq.ml])
t += seq.ml
continue
}
}
// We must be in current buffer now
if seq.ml > 0 {
start := t - seq.mo
if seq.ml <= t-start {
// No overlap
copy(out[t:], out[start:start+seq.ml])
t += seq.ml
continue
} else {
// Overlapping copy
// Extend destination slice and copy one byte at the time.
src := out[start : start+seq.ml]
dst := out[t:]
dst = dst[:len(src)]
t += len(src)
// Destination is the space we just added.
for i := range src {
dst[i] = src[i]
}
}
}
}
// Add final literals
copy(out[t:], s.literals)
if debugDecoder {
t += len(s.literals)
if t != len(out) {
panic(fmt.Errorf("length mismatch, want %d, got %d, ss: %d", len(out), t, s.seqSize))
}
}
s.out = out
return nil
}
// decode sequences from the stream with the provided history.
func (s *sequenceDecs) decodeSync(history *history) error {
br := s.br
seqs := s.nSeqs
startSize := len(s.out) startSize := len(s.out)
// Grab full sizes tables, to avoid bounds checks. // Grab full sizes tables, to avoid bounds checks.
llTable, mlTable, ofTable := s.litLengths.fse.dt[:maxTablesize], s.matchLengths.fse.dt[:maxTablesize], s.offsets.fse.dt[:maxTablesize] llTable, mlTable, ofTable := s.litLengths.fse.dt[:maxTablesize], s.matchLengths.fse.dt[:maxTablesize], s.offsets.fse.dt[:maxTablesize]
llState, mlState, ofState := s.litLengths.state.state, s.matchLengths.state.state, s.offsets.state.state llState, mlState, ofState := s.litLengths.state.state, s.matchLengths.state.state, s.offsets.state.state
hist := history.b[history.ignoreBuffer:]
out := s.out
maxBlockSize := maxCompressedBlockSize
if s.windowSize < maxBlockSize {
maxBlockSize = s.windowSize
}
for i := seqs - 1; i >= 0; i-- { for i := seqs - 1; i >= 0; i-- {
if br.overread() { if br.overread() {
@ -151,7 +406,7 @@ func (s *sequenceDecs) decode(seqs int, br *bitReader, hist []byte) error {
if temp == 0 { if temp == 0 {
// 0 is not valid; input is corrupted; force offset to 1 // 0 is not valid; input is corrupted; force offset to 1
println("temp was 0") println("WARNING: temp was 0")
temp = 1 temp = 1
} }
@ -176,51 +431,49 @@ func (s *sequenceDecs) decode(seqs int, br *bitReader, hist []byte) error {
if ll > len(s.literals) { if ll > len(s.literals) {
return fmt.Errorf("unexpected literal count, want %d bytes, but only %d is available", ll, len(s.literals)) return fmt.Errorf("unexpected literal count, want %d bytes, but only %d is available", ll, len(s.literals))
} }
size := ll + ml + len(s.out) size := ll + ml + len(out)
if size-startSize > maxBlockSize { if size-startSize > maxBlockSize {
return fmt.Errorf("output (%d) bigger than max block size", size) return fmt.Errorf("output (%d) bigger than max block size (%d)", size, maxBlockSize)
} }
if size > cap(s.out) { if size > cap(out) {
// Not enough size, which can happen under high volume block streaming conditions // Not enough size, which can happen under high volume block streaming conditions
// but could be if destination slice is too small for sync operations. // but could be if destination slice is too small for sync operations.
// over-allocating here can create a large amount of GC pressure so we try to keep // over-allocating here can create a large amount of GC pressure so we try to keep
// it as contained as possible // it as contained as possible
used := len(s.out) - startSize used := len(out) - startSize
addBytes := 256 + ll + ml + used>>2 addBytes := 256 + ll + ml + used>>2
// Clamp to max block size. // Clamp to max block size.
if used+addBytes > maxBlockSize { if used+addBytes > maxBlockSize {
addBytes = maxBlockSize - used addBytes = maxBlockSize - used
} }
s.out = append(s.out, make([]byte, addBytes)...) out = append(out, make([]byte, addBytes)...)
s.out = s.out[:len(s.out)-addBytes] out = out[:len(out)-addBytes]
} }
if ml > maxMatchLen { if ml > maxMatchLen {
return fmt.Errorf("match len (%d) bigger than max allowed length", ml) return fmt.Errorf("match len (%d) bigger than max allowed length", ml)
} }
// Add literals // Add literals
s.out = append(s.out, s.literals[:ll]...) out = append(out, s.literals[:ll]...)
s.literals = s.literals[ll:] s.literals = s.literals[ll:]
out := s.out
if mo == 0 && ml > 0 { if mo == 0 && ml > 0 {
return fmt.Errorf("zero matchoff and matchlen (%d) > 0", ml) return fmt.Errorf("zero matchoff and matchlen (%d) > 0", ml)
} }
if mo > len(s.out)+len(hist) || mo > s.windowSize { if mo > len(out)+len(hist) || mo > s.windowSize {
if len(s.dict) == 0 { if len(s.dict) == 0 {
return fmt.Errorf("match offset (%d) bigger than current history (%d)", mo, len(s.out)+len(hist)) return fmt.Errorf("match offset (%d) bigger than current history (%d)", mo, len(out)+len(hist))
} }
// we may be in dictionary. // we may be in dictionary.
dictO := len(s.dict) - (mo - (len(s.out) + len(hist))) dictO := len(s.dict) - (mo - (len(out) + len(hist)))
if dictO < 0 || dictO >= len(s.dict) { if dictO < 0 || dictO >= len(s.dict) {
return fmt.Errorf("match offset (%d) bigger than current history (%d)", mo, len(s.out)+len(hist)) return fmt.Errorf("match offset (%d) bigger than current history (%d)", mo, len(out)+len(hist))
} }
end := dictO + ml end := dictO + ml
if end > len(s.dict) { if end > len(s.dict) {
out = append(out, s.dict[dictO:]...) out = append(out, s.dict[dictO:]...)
mo -= len(s.dict) - dictO
ml -= len(s.dict) - dictO ml -= len(s.dict) - dictO
} else { } else {
out = append(out, s.dict[dictO:end]...) out = append(out, s.dict[dictO:end]...)
@ -231,26 +484,25 @@ func (s *sequenceDecs) decode(seqs int, br *bitReader, hist []byte) error {
// Copy from history. // Copy from history.
// TODO: Blocks without history could be made to ignore this completely. // TODO: Blocks without history could be made to ignore this completely.
if v := mo - len(s.out); v > 0 { if v := mo - len(out); v > 0 {
// v is the start position in history from end. // v is the start position in history from end.
start := len(s.hist) - v start := len(hist) - v
if ml > v { if ml > v {
// Some goes into current block. // Some goes into current block.
// Copy remainder of history // Copy remainder of history
out = append(out, s.hist[start:]...) out = append(out, hist[start:]...)
mo -= v
ml -= v ml -= v
} else { } else {
out = append(out, s.hist[start:start+ml]...) out = append(out, hist[start:start+ml]...)
ml = 0 ml = 0
} }
} }
// We must be in current buffer now // We must be in current buffer now
if ml > 0 { if ml > 0 {
start := len(s.out) - mo start := len(out) - mo
if ml <= len(s.out)-start { if ml <= len(out)-start {
// No overlap // No overlap
out = append(out, s.out[start:start+ml]...) out = append(out, out[start:start+ml]...)
} else { } else {
// Overlapping copy // Overlapping copy
// Extend destination slice and copy one byte at the time. // Extend destination slice and copy one byte at the time.
@ -264,7 +516,6 @@ func (s *sequenceDecs) decode(seqs int, br *bitReader, hist []byte) error {
} }
} }
} }
s.out = out
if i == 0 { if i == 0 {
// This is the last sequence, so we shouldn't update state. // This is the last sequence, so we shouldn't update state.
break break
@ -291,9 +542,14 @@ func (s *sequenceDecs) decode(seqs int, br *bitReader, hist []byte) error {
} }
} }
// Check if space for literals
if len(s.literals)+len(s.out)-startSize > maxBlockSize {
return fmt.Errorf("output (%d) bigger than max block size (%d)", len(s.out), maxBlockSize)
}
// Add final literals // Add final literals
s.out = append(s.out, s.literals...) s.out = append(out, s.literals...)
return nil return br.close()
} }
// update states, at least 27 bits must be available. // update states, at least 27 bits must be available.
@ -457,36 +713,3 @@ func (s *sequenceDecs) adjustOffset(offset, litLen int, offsetB uint8) int {
s.prevOffset[0] = temp s.prevOffset[0] = temp
return temp return temp
} }
// mergeHistory will merge history.
func (s *sequenceDecs) mergeHistory(hist *sequenceDecs) (*sequenceDecs, error) {
for i := uint(0); i < 3; i++ {
var sNew, sHist *sequenceDec
switch i {
default:
// same as "case 0":
sNew = &s.litLengths
sHist = &hist.litLengths
case 1:
sNew = &s.offsets
sHist = &hist.offsets
case 2:
sNew = &s.matchLengths
sHist = &hist.matchLengths
}
if sNew.repeat {
if sHist.fse == nil {
return nil, fmt.Errorf("sequence stream %d, repeat requested, but no history", i)
}
continue
}
if sNew.fse == nil {
return nil, fmt.Errorf("sequence stream %d, no fse found", i)
}
if sHist.fse != nil && !sHist.fse.preDefined {
fseDecoderPool.Put(sHist.fse)
}
sHist.fse = sNew.fse
}
return hist, nil
}

18
vendor/github.com/klauspost/compress/zstd/zip.go generated vendored
View File

@ -20,7 +20,7 @@ const ZipMethodPKWare = 20
var zipReaderPool sync.Pool var zipReaderPool sync.Pool
// newZipReader cannot be used since we would leak goroutines... // newZipReader creates a pooled zip decompressor.
func newZipReader(r io.Reader) io.ReadCloser { func newZipReader(r io.Reader) io.ReadCloser {
dec, ok := zipReaderPool.Get().(*Decoder) dec, ok := zipReaderPool.Get().(*Decoder)
if ok { if ok {
@ -44,10 +44,14 @@ func (r *pooledZipReader) Read(p []byte) (n int, err error) {
r.mu.Lock() r.mu.Lock()
defer r.mu.Unlock() defer r.mu.Unlock()
if r.dec == nil { if r.dec == nil {
return 0, errors.New("Read after Close") return 0, errors.New("read after close or EOF")
} }
dec, err := r.dec.Read(p) dec, err := r.dec.Read(p)
if err == io.EOF {
err = r.dec.Reset(nil)
zipReaderPool.Put(r.dec)
r.dec = nil
}
return dec, err return dec, err
} }
@ -112,11 +116,5 @@ func ZipCompressor(opts ...EOption) func(w io.Writer) (io.WriteCloser, error) {
// ZipDecompressor returns a decompressor that can be registered with zip libraries. // ZipDecompressor returns a decompressor that can be registered with zip libraries.
// See ZipCompressor for example. // See ZipCompressor for example.
func ZipDecompressor() func(r io.Reader) io.ReadCloser { func ZipDecompressor() func(r io.Reader) io.ReadCloser {
return func(r io.Reader) io.ReadCloser { return newZipReader
d, err := NewReader(r, WithDecoderConcurrency(1), WithDecoderLowmem(true))
if err != nil {
panic(err)
}
return d.IOReadCloser()
}
} }

11
vendor/github.com/klauspost/compress/zstd/zstd.go generated vendored
View File

@ -39,6 +39,9 @@ const zstdMinMatch = 3
// Reset the buffer offset when reaching this. // Reset the buffer offset when reaching this.
const bufferReset = math.MaxInt32 - MaxWindowSize const bufferReset = math.MaxInt32 - MaxWindowSize
// fcsUnknown is used for unknown frame content size.
const fcsUnknown = math.MaxUint64
var ( var (
// ErrReservedBlockType is returned when a reserved block type is found. // ErrReservedBlockType is returned when a reserved block type is found.
// Typically this indicates wrong or corrupted input. // Typically this indicates wrong or corrupted input.
@ -52,6 +55,10 @@ var (
// Typically returned on invalid input. // Typically returned on invalid input.
ErrBlockTooSmall = errors.New("block too small") ErrBlockTooSmall = errors.New("block too small")
// ErrUnexpectedBlockSize is returned when a block has unexpected size.
// Typically returned on invalid input.
ErrUnexpectedBlockSize = errors.New("unexpected block size")
// ErrMagicMismatch is returned when a "magic" number isn't what is expected. // ErrMagicMismatch is returned when a "magic" number isn't what is expected.
// Typically this indicates wrong or corrupted input. // Typically this indicates wrong or corrupted input.
ErrMagicMismatch = errors.New("invalid input: magic number mismatch") ErrMagicMismatch = errors.New("invalid input: magic number mismatch")
@ -75,6 +82,10 @@ var (
// This is only returned if SingleSegment is specified on the frame. // This is only returned if SingleSegment is specified on the frame.
ErrFrameSizeExceeded = errors.New("frame size exceeded") ErrFrameSizeExceeded = errors.New("frame size exceeded")
// ErrFrameSizeMismatch is returned if the stated frame size does not match the expected size.
// This is only returned if SingleSegment is specified on the frame.
ErrFrameSizeMismatch = errors.New("frame size does not match size on stream")
// ErrCRCMismatch is returned if CRC mismatches. // ErrCRCMismatch is returned if CRC mismatches.
ErrCRCMismatch = errors.New("CRC check failed") ErrCRCMismatch = errors.New("CRC check failed")

45
vendor/github.com/lrstanley/girc/.editorconfig generated vendored Normal file
View File

@ -0,0 +1,45 @@
# THIS FILE IS GENERATED! DO NOT EDIT! Maintained by Terraform.
#
# editorconfig.org
root = true
[*]
charset = utf-8
end_of_line = lf
indent_size = 4
indent_style = space
insert_final_newline = true
trim_trailing_whitespace = true
[*.tf]
indent_size = 2
[*.go]
indent_style = tab
indent_size = 4
[*.md]
trim_trailing_whitespace = false
[*.{md,py,sh,yml,yaml}]
max_line_length = 105
[*.{yml,yaml,toml}]
indent_size = 2
[*.json]
indent_size = 2
insert_final_newline = ignore
[*.html]
indent_size = 2
[Makefile]
indent_style = tab
[**.min.js]
indent_style = ignore
insert_final_newline = ignore
[*.bat]
indent_style = tab

37
vendor/github.com/lrstanley/girc/.golangci.yml generated vendored Normal file
View File

@ -0,0 +1,37 @@
# THIS FILE IS GENERATED! DO NOT EDIT! Maintained by Terraform.
run:
tests: False
timeout: 3m
issues:
max-per-linter: 0
max-same-issues: 0
severity:
default-severity: error
rules:
- linters:
- errcheck
- gocritic
severity: warning
linters:
enable:
- asciicheck
- exportloopref
- gci
- gocritic
- gofmt
- misspell
linters-settings:
gocritic:
disabled-checks:
- hugeParam
enabled-tags:
- diagnostic
- opinionated
- performance
- style
govet:
check-shadowing: true

25
vendor/github.com/lrstanley/girc/.travis.yml generated vendored
View File

@ -1,25 +0,0 @@
language: go
go:
- 1.11.x
- tip
before_install:
- go get -v golang.org/x/lint/golint
script:
- $HOME/gopath/bin/golint -min_confidence 0.9 -set_exit_status
- GORACE="exitcode=1 halt_on_error=1" go test -v -coverprofile=coverage.txt -race -timeout 3m -count 3 -cpu 1,4
- go vet -v .
after_success:
- bash <(curl -s https://codecov.io/bash)
branches:
only:
- master
notifications:
irc:
channels:
- irc.byteirc.org#/dev/null
template:
- "%{repository} #%{build_number} %{branch}/%{commit}: %{author} -- %{message}
%{build_url}"
on_success: change
on_failure: change
skip_join: false

122
vendor/github.com/lrstanley/girc/CODE_OF_CONDUCT.md generated vendored Normal file
View File

@ -0,0 +1,122 @@
<!-- THIS FILE IS GENERATED! DO NOT EDIT! Maintained by Terraform. -->
# Code of Conduct
## Our Pledge :purple_heart:
We as members, contributors, and leaders pledge to make participation in our
community a harassment-free experience for everyone, regardless of age, body
size, visible or invisible disability, ethnicity, sex characteristics, gender
identity and expression, level of experience, education, socio-economic status,
nationality, personal appearance, race, caste, color, religion, or sexual
identity and orientation.
We pledge to act and interact in ways that contribute to an open, welcoming,
diverse, inclusive, and healthy community.
## Our Standards
Examples of behavior that contributes to a positive environment for our
community include:
* Demonstrating empathy and kindness toward other people
* Being respectful of differing opinions, viewpoints, and experiences
* Giving and gracefully accepting constructive feedback
* Accepting responsibility and apologizing to those affected by our mistakes,
and learning from the experience
* Focusing on what is best not just for us as individuals, but for the overall
community
Examples of unacceptable behavior include:
* The use of sexualized language or imagery, and sexual attention or advances of
any kind
* Trolling, insulting or derogatory comments, and personal or political attacks
* Public or private harassment
* Publishing others' private information, such as a physical or email address,
without their explicit permission
* Other conduct which could reasonably be considered inappropriate in a
professional setting
## Enforcement Responsibilities
Community leaders are responsible for clarifying and enforcing our standards of
acceptable behavior and will take appropriate and fair corrective action in
response to any behavior that they deem inappropriate, threatening, offensive,
or harmful.
Community leaders have the right and responsibility to remove, edit, or reject
comments, commits, code, wiki edits, issues, and other contributions that are
not aligned to this Code of Conduct, and will communicate reasons for moderation
decisions when appropriate.
## Scope
This Code of Conduct applies within all community spaces, and also applies when
an individual is officially representing the community in public spaces.
Examples of representing our community include using an official e-mail address,
posting via an official social media account, or acting as an appointed
representative at an online or offline event.
## Enforcement
Instances of abusive, harassing, or otherwise unacceptable behavior may be
reported to the community leaders responsible for enforcement at
disclosure@liamstanley.io. All complaints will be reviewed and investigated
promptly and fairly.
All community leaders are obligated to respect the privacy and security of the
reporter of any incident.
## Enforcement Guidelines
Community leaders will follow these Community Impact Guidelines in determining
the consequences for any action they deem in violation of this Code of Conduct:
### 1. Correction
**Community Impact**: Use of inappropriate language or other behavior deemed
unprofessional or unwelcome in the community.
**Consequence**: A private, written warning from community leaders, providing
clarity around the nature of the violation and an explanation of why the
behavior was inappropriate. A public apology may be requested.
### 2. Warning
**Community Impact**: A violation through a single incident or series of
actions.
**Consequence**: A warning with consequences for continued behavior. No
interaction with the people involved, including unsolicited interaction with
those enforcing the Code of Conduct, for a specified period of time. This
includes avoiding interactions in community spaces as well as external channels
like social media. Violating these terms may lead to a temporary or permanent
ban.
### 3. Temporary Ban
**Community Impact**: A serious violation of community standards, including
sustained inappropriate behavior.
**Consequence**: A temporary ban from any sort of interaction or public
communication with the community for a specified period of time. No public or
private interaction with the people involved, including unsolicited interaction
with those enforcing the Code of Conduct, is allowed during this period.
Violating these terms may lead to a permanent ban.
### 4. Permanent Ban
**Community Impact**: Demonstrating a pattern of violation of community
standards, including sustained inappropriate behavior, harassment of an
individual, or aggression toward or disparagement of classes of individuals.
**Consequence**: A permanent ban from any sort of public interaction within the
community.
## Attribution
This Code of Conduct is adapted from the Contributor Covenant,
version 2.1, available [here](https://www.contributor-covenant.org/version/2/1/code_of_conduct.html).
For answers to common questions about this code of conduct, see the [FAQ](https://www.contributor-covenant.org/faq).
Translations are available at [translations](https://www.contributor-covenant.org/translations).

116
vendor/github.com/lrstanley/girc/CONTRIBUTING.md generated vendored
View File

@ -1,32 +1,106 @@
# Contributing <!-- THIS FILE IS GENERATED! DO NOT EDIT! Maintained by Terraform. -->
# :handshake: Contributing
## Issue submission This document outlines some of the guidelines that we try and adhere to while
working on this project.
* When submitting an issue or bug report, please ensure to provide as much > :point_right: **Note**: before participating in the community, please read our
information as possible, please ensure that you are running on the latest > [Code of Conduct][coc].
stable version (tagged), or when using master, provide the specific commit > By interacting with this repository, organization, or community you agree to
being used. > abide by our Code of Conduct.
>
> Additionally, if you contribute **any source code** to this repository, you
> agree to the terms of the [Developer Certificate of Origin][dco]. This helps
> ensure that contributions aren't in violation of 3rd party license terms.
## :lady_beetle: Issue submission
When [submitting an issue][issues] or bug report,
please follow these guidelines:
* Provide as much information as possible (logs, metrics, screenshots,
runtime environment, etc).
* Ensure that you are running on the latest stable version (tagged), or
when using `master`, provide the specific commit being used.
* Provide the minimum needed viable source to replicate the problem. * Provide the minimum needed viable source to replicate the problem.
## Pull requests ## :bulb: Feature requests
When [submitting a feature request][issues], please
follow these guidelines:
* Does this feature benefit others? or just your usecase? If the latter,
it will likely be declined, unless it has a more broad benefit to others.
* Please include the pros and cons of the feature.
* If possible, describe how the feature would work, and any diagrams/mock
examples of what the feature would look like.
## :rocket: Pull requests
To review what is currently being worked on, or looked into, feel free to head To review what is currently being worked on, or looked into, feel free to head
over to the [issues list](../../issues). over to the [open pull requests][pull-requests] or [issues list][issues].
Below are a few guidelines if you would like to contribute. Keep the code ## :raised_back_of_hand: Assistance with discussions
clean, standardized, and much of the quality should match Golang's standard
library and common idioms.
* Always test using the latest Go version. * Take a look at the [open discussions][discussions], and if you feel like
* Always use `gofmt` before committing anything. you'd like to help out other members of the community, it would be much
* Always have proper documentation before committing. appreciated!
* Keep the same whitespacing, documentation, and newline format as the
rest of the project. ## :pushpin: Guidelines
* Only use 3rd party libraries if necessary. If only a small portion of
### :test_tube: Language agnostic
Below are a few guidelines if you would like to contribute:
* If the feature is large or the bugfix has potential breaking changes,
please open an issue first to ensure the changes go down the best path.
* If possible, break the changes into smaller PRs. Pull requests should be
focused on a specific feature/fix.
* Pull requests will only be accepted with sufficient documentation
describing the new functionality/fixes.
* Keep the code simple where possible. Code that is smaller/more compact
does not mean better. Don't do magic behind the scenes.
* Use the same formatting/styling/structure as existing code.
* Follow idioms and community-best-practices of the related language,
unless the previous above guidelines override what the community
recommends.
* Always test your changes, both the features/fixes being implemented, but
also in the standard way that a user would use the project (not just
your configuration that fixes your issue).
* Only use 3rd party libraries when necessary. If only a small portion of
the library is needed, simply rewrite it within the library to prevent the library is needed, simply rewrite it within the library to prevent
useless imports. useless imports.
* Also see [golang/go/wiki/CodeReviewComments](https://github.com/golang/go/wiki/CodeReviewComments)
If you would like to assist, and the pull request is quite large and/or it has ### :hamster: Golang
the potential of being a breaking change, please open an issue first so it can
be discussed. * See [golang/go/wiki/CodeReviewComments](https://github.com/golang/go/wiki/CodeReviewComments)
* This project uses [golangci-lint](https://golangci-lint.run/) for
Go-related files. This should be available for any editor that supports
`gopls`, however you can also run it locally with `golangci-lint run`
after installing it.
## :clipboard: References
* [Open Source: How to Contribute](https://opensource.guide/how-to-contribute/)
* [About pull requests](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/about-pull-requests)
* [GitHub Docs](https://docs.github.com/)
## :speech_balloon: What to do next?
* :old_key: Find a vulnerability? Check out our [Security and Disclosure][security] policy.
* :link: Repository [License][license].
* [Support][support]
* [Code of Conduct][coc].
<!-- definitions -->
[coc]: https://github.com/lrstanley/girc/blob/master/CODE_OF_CONDUCT.md
[dco]: https://developercertificate.org/
[discussions]: https://github.com/lrstanley/girc/discussions
[issues]: https://github.com/lrstanley/girc/issues/new/choose
[license]: https://github.com/lrstanley/girc/blob/master/LICENSE
[pull-requests]: https://github.com/lrstanley/girc/issues/new/choose
[security]: https://github.com/lrstanley/girc/security/policy
[support]: https://github.com/lrstanley/girc/blob/master/SUPPORT.md

122
vendor/github.com/lrstanley/girc/README.md generated vendored
View File

@ -1,12 +1,49 @@
<p align="center"><a href="https://pkg.go.dev/github.com/lrstanley/girc"><img width="270" src="http://i.imgur.com/DEnyrdB.png"></a></p> <p align="center"><a href="https://pkg.go.dev/github.com/lrstanley/girc"><img width="270" src="http://i.imgur.com/DEnyrdB.png"></a></p>
<p align="center">girc, a flexible IRC library for Go</p> <!-- template:begin:header -->
<!-- do not edit anything in this "template" block, its auto-generated -->
<p align="center">girc -- :bomb: girc is a flexible IRC library for Go :ok_hand:</p>
<p align="center"> <p align="center">
<a href="https://github.com/lrstanley/girc/actions"><img src="https://github.com/lrstanley/girc/workflows/test/badge.svg" alt="Test Status"></a>
<a href="https://codecov.io/gh/lrstanley/girc"><img src="https://codecov.io/gh/lrstanley/girc/branch/master/graph/badge.svg" alt="Coverage Status"></a> <a href="https://github.com/lrstanley/girc/actions?query=workflow%3Atest+event%3Apush">
<a href="https://pkg.go.dev/github.com/lrstanley/girc"><img src="https://pkg.go.dev/badge/github.com/lrstanley/girc" alt="GoDoc"></a> <img alt="GitHub Workflow Status (test @ master)" src="https://img.shields.io/github/workflow/status/lrstanley/girc/test/master?label=test&style=flat-square&event=push">
<a href="https://goreportcard.com/report/github.com/lrstanley/girc"><img src="https://goreportcard.com/badge/github.com/lrstanley/girc" alt="Go Report Card"></a> </a>
<a href="https://liam.sh/chat"><img src="https://img.shields.io/badge/community-chat%20with%20us-green.svg" alt="Community Chat"></a>
<img alt="Code Coverage" src="https://img.shields.io/codecov/c/github/lrstanley/girc/master?style=flat-square">
<a href="https://pkg.go.dev/github.com/lrstanley/girc">
<img alt="Go Documentation" src="https://pkg.go.dev/badge/github.com/lrstanley/girc?style=flat-square">
</a>
<a href="https://goreportcard.com/report/github.com/lrstanley/girc">
<img alt="Go Report Card" src="https://goreportcard.com/badge/github.com/lrstanley/girc?style=flat-square">
</a>
<img alt="Bug reports" src="https://img.shields.io/github/issues/lrstanley/girc/bug?label=issues&style=flat-square">
<img alt="Feature requests" src="https://img.shields.io/github/issues/lrstanley/girc/enhancement?label=feature%20requests&style=flat-square">
<a href="https://github.com/lrstanley/girc/pulls">
<img alt="Open Pull Requests" src="https://img.shields.io/github/issues-pr/lrstanley/girc?style=flat-square">
</a>
<a href="https://github.com/lrstanley/girc/tags">
<img alt="Latest Semver Tag" src="https://img.shields.io/github/v/tag/lrstanley/girc?style=flat-square">
</a>
<img alt="Last commit" src="https://img.shields.io/github/last-commit/lrstanley/girc?style=flat-square">
<a href="https://github.com/lrstanley/girc/discussions/new?category=q-a">
<img alt="Ask a Question" src="https://img.shields.io/badge/discussions-ask_a_question!-green?style=flat-square">
</a>
<a href="https://liam.sh/chat"><img src="https://img.shields.io/badge/discord-bytecord-blue.svg?style=flat-square" alt="Discord Chat"></a>
</p> </p>
<!-- template:end:header -->
<!-- template:begin:toc -->
<!-- do not edit anything in this "template" block, its auto-generated -->
## :link: Table of Contents
- [Features](#features)
- [Installing](#installing)
- [Examples](#examples)
- [References](#references)
- [Support & Assistance](#raising_hand_man-support-assistance)
- [Contributing](#handshake-contributing)
- [License](#balance_scale-license)
<!-- template:end:toc -->
## Features ## Features
@ -48,12 +85,54 @@ usecases/examples/projects which utilize girc:
Working on a project and want to add it to the list? Submit a pull request! Working on a project and want to add it to the list? Submit a pull request!
## Contributing ## References
Please review the [CONTRIBUTING](CONTRIBUTING.md) doc for submitting issues/a guide * [IRCv3: Specification Docs](http://ircv3.net/irc/)
on submitting pull requests and helping out. * [IRCv3: Specification Repo](https://github.com/ircv3/ircv3-specifications)
* [IRCv3 Capability Registry](http://ircv3.net/registry.html)
* [IRCv3: WEBIRC](https://ircv3.net/specs/extensions/webirc.html)
* [KiwiIRC: WEBIRC](https://kiwiirc.com/docs/webirc)
* [ISUPPORT Specification Docs](http://www.irc.org/tech_docs/005.html) ([alternative 1](http://defs.ircdocs.horse/defs/isupport.html), [alternative 2](https://github.com/grawity/irc-docs/blob/master/client/RPL_ISUPPORT/draft-hardy-irc-isupport-00.txt), [relevant draft](http://www.irc.org/tech_docs/draft-brocklesby-irc-isupport-03.txt))
* [IRC Numerics List](http://defs.ircdocs.horse/defs/numerics.html)
* [Extended WHO (also known as WHOX)](https://github.com/quakenet/snircd/blob/master/doc/readme.who)
* [RFC1459: Internet Relay Chat Protocol](https://tools.ietf.org/html/rfc1459)
* [RFC2812: Internet Relay Chat: Client Protocol](https://tools.ietf.org/html/rfc2812)
* [RFC2813: Internet Relay Chat: Server Protocol](https://tools.ietf.org/html/rfc2813)
* [RFC7194: Default Port for Internet Relay Chat (IRC) via TLS/SSL](https://tools.ietf.org/html/rfc7194)
* [RFC4422: Simple Authentication and Security Layer](https://tools.ietf.org/html/rfc4422) ([SASL EXTERNAL](https://tools.ietf.org/html/rfc4422#appendix-A))
* [RFC4616: The PLAIN SASL Mechanism](https://tools.ietf.org/html/rfc4616)
## License
<!-- template:begin:support -->
<!-- do not edit anything in this "template" block, its auto-generated -->
## :raising_hand_man: Support & Assistance
* :heart: Please review the [Code of Conduct](CODE_OF_CONDUCT.md) for
guidelines on ensuring everyone has the best experience interacting with
the community.
* :raising_hand_man: Take a look at the [support](SUPPORT.md) document on
guidelines for tips on how to ask the right questions.
* :lady_beetle: For all features/bugs/issues/questions/etc, [head over here](https://github.com/lrstanley/girc/issues/new/choose).
<!-- template:end:support -->
<!-- template:begin:contributing -->
<!-- do not edit anything in this "template" block, its auto-generated -->
## :handshake: Contributing
* :heart: Please review the [Code of Conduct](CODE_OF_CONDUCT.md) for guidelines
on ensuring everyone has the best experience interacting with the
community.
* :clipboard: Please review the [contributing](CONTRIBUTING.md) doc for submitting
issues/a guide on submitting pull requests and helping out.
* :old_key: For anything security related, please review this repositories [security policy](https://github.com/lrstanley/girc/security/policy).
<!-- template:end:contributing -->
<!-- template:begin:license -->
<!-- do not edit anything in this "template" block, its auto-generated -->
## :balance_scale: License
```
MIT License
Copyright (c) 2016 Liam Stanley <me@liamstanley.io> Copyright (c) 2016 Liam Stanley <me@liamstanley.io>
@ -74,22 +153,9 @@ on submitting pull requests and helping out.
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE. SOFTWARE.
```
girc artwork licensed under [CC 3.0](http://creativecommons.org/licenses/by/3.0/) based on Renee French under Creative Commons 3.0 Attributions. _Also located [here](LICENSE)_
<!-- template:end:license -->
## References girc artwork licensed under [CC 3.0](http://creativecommons.org/licenses/by/3.0/)
based on Renee French under Creative Commons 3.0 Attributions.
* [IRCv3: Specification Docs](http://ircv3.net/irc/)
* [IRCv3: Specification Repo](https://github.com/ircv3/ircv3-specifications)
* [IRCv3 Capability Registry](http://ircv3.net/registry.html)
* [IRCv3: WEBIRC](https://ircv3.net/specs/extensions/webirc.html)
* [KiwiIRC: WEBIRC](https://kiwiirc.com/docs/webirc)
* [ISUPPORT Specification Docs](http://www.irc.org/tech_docs/005.html) ([alternative 1](http://defs.ircdocs.horse/defs/isupport.html), [alternative 2](https://github.com/grawity/irc-docs/blob/master/client/RPL_ISUPPORT/draft-hardy-irc-isupport-00.txt), [relevant draft](http://www.irc.org/tech_docs/draft-brocklesby-irc-isupport-03.txt))
* [IRC Numerics List](http://defs.ircdocs.horse/defs/numerics.html)
* [Extended WHO (also known as WHOX)](https://github.com/quakenet/snircd/blob/master/doc/readme.who)
* [RFC1459: Internet Relay Chat Protocol](https://tools.ietf.org/html/rfc1459)
* [RFC2812: Internet Relay Chat: Client Protocol](https://tools.ietf.org/html/rfc2812)
* [RFC2813: Internet Relay Chat: Server Protocol](https://tools.ietf.org/html/rfc2813)
* [RFC7194: Default Port for Internet Relay Chat (IRC) via TLS/SSL](https://tools.ietf.org/html/rfc7194)
* [RFC4422: Simple Authentication and Security Layer](https://tools.ietf.org/html/rfc4422) ([SASL EXTERNAL](https://tools.ietf.org/html/rfc4422#appendix-A))
* [RFC4616: The PLAIN SASL Mechanism](https://tools.ietf.org/html/rfc4616)

54
vendor/github.com/lrstanley/girc/SECURITY.md generated vendored Normal file
View File

@ -0,0 +1,54 @@
<!-- THIS FILE IS GENERATED! DO NOT EDIT! Maintained by Terraform. -->
# :old_key: Security Policy
## :heavy_check_mark: Supported Versions
The following restrictions apply for versions that are still supported in terms of security and bug fixes:
* :grey_question: Must be using the latest major/minor version.
* :grey_question: Must be using a supported platform for the repository (e.g. OS, browser, etc), and that platform must
be within its supported versions (for example: don't use a legacy or unsupported version of Ubuntu or
Google Chrome).
* :grey_question: Repository must not be archived (unless the vulnerability is critical, and the repository moderately
popular).
* :heavy_check_mark:
If one of the above doesn't apply to you, feel free to submit an issue and we can discuss the
issue/vulnerability further.
## :lady_beetle: Reporting a Vulnerability
Best method of contact: [GPG :key:](https://github.com/lrstanley.gpg)
* :speech_balloon: [Discord][chat]: message `/home/liam#0000`.
* :email: Email: `security@liamstanley.io`
Backup contacts (if I am unresponsive after **48h**): [GPG :key:](https://github.com/FM1337.gpg)
* :speech_balloon: [Discord][chat]: message `Allen#7440`.
* :email: Email: `security@allenlydiard.ca`
If you feel that this disclosure doesn't include a critical vulnerability and there is no sensitive
information in the disclosure, you don't have to use the GPG key. For all other situations, please
use it.
### :stopwatch: Vulnerability disclosure expectations
* :no_bell: We expect you to not share this information with others, unless:
* The maximum timeline for initial response has been exceeded (shown below).
* The maximum resolution time has been exceeded (shown below).
* :mag_right: We expect you to responsibly investigate this vulnerability -- please do not utilize the
vulnerability beyond the initial findings.
* :stopwatch: Initial response within 48h, however, if the primary contact shown above is unavailable, please
use the backup contacts provided. The maximum timeline for an initial response should be within
7 days.
* :stopwatch: Depending on the severity of the disclosure, resolution time may be anywhere from 24h to 2
weeks after initial response, though in most cases it will likely be closer to the former.
* If the vulnerability is very low/low in terms of risk, the above timelines **will not apply**.
* :toolbox: Before the release of resolved versions, a [GitHub Security Advisory][advisory-docs].
will be released on the respective repository. [Browser all advisories here][advisory].
<!-- definitions -->
[chat]: https://liam.sh/chat
[advisory]: https://github.com/advisories?query=type%3Areviewed+ecosystem%3Ago
[advisory-docs]: https://docs.github.com/en/code-security/repository-security-advisories/creating-a-repository-security-advisory

56
vendor/github.com/lrstanley/girc/SUPPORT.md generated vendored Normal file
View File

@ -0,0 +1,56 @@
# :raising_hand_man: Support
This document explains where and how to get help with most of my projects.
Please ensure you read through it thoroughly.
> :point_right: **Note**: before participating in the community, please read our
> [Code of Conduct][coc].
> By interacting with this repository, organization, or community you agree to
> abide by its terms.
## :grey_question: Asking quality questions
Questions can go to [Github Discussions][discussions] or feel free to join
the Discord [here][chat].
Help me help you! Spend time framing questions and add links and resources.
Spending the extra time up front can help save everyone time in the long run.
Here are some tips:
* Don't fall for the [XY problem][xy].
* Search to find out if a similar question has been asked or if a similar
issue/bug has been reported.
* Try to define what you need help with:
* Is there something in particular you want to do?
* What problem are you encountering and what steps have you taken to try
and fix it?
* Is there a concept you don't understand?
* Provide sample code, such as a [CodeSandbox][cs] or a simple snippet, if
possible.
* Screenshots can help, but if there's important text such as code or error
messages in them, please also provide those.
* The more time you put into asking your question, the better I and others
can help you.
## :old_key: Security
For any security or vulnerability related disclosure, please follow the
guidelines outlined in our [security policy][security].
## :handshake: Contributions
See [`CONTRIBUTING.md`][contributing] on how to contribute.
<!-- definitions -->
[coc]: https://github.com/lrstanley/girc/blob/master/CODE_OF_CONDUCT.md
[contributing]: https://github.com/lrstanley/girc/blob/master/CONTRIBUTING.md
[discussions]: https://github.com/lrstanley/girc/discussions/categories/q-a
[issues]: https://github.com/lrstanley/girc/issues/new/choose
[license]: https://github.com/lrstanley/girc/blob/master/LICENSE
[pull-requests]: https://github.com/lrstanley/girc/issues/new/choose
[security]: https://github.com/lrstanley/girc/security/policy
[support]: https://github.com/lrstanley/girc/blob/master/SUPPORT.md
[xy]: https://meta.stackexchange.com/questions/66377/what-is-the-xy-problem/66378#66378
[chat]: https://liam.sh/chat
[cs]: https://codesandbox.io

2
vendor/github.com/lrstanley/girc/builtin.go generated vendored
View File

@ -427,7 +427,7 @@ func handleMOTD(c *Client, e Event) {
} }
// Otherwise, assume we're getting sent the MOTD line-by-line. // Otherwise, assume we're getting sent the MOTD line-by-line.
if len(c.state.motd) != 0 { if c.state.motd != "" {
c.state.motd += "\n" c.state.motd += "\n"
} }
c.state.motd += e.Last() c.state.motd += e.Last()

2
vendor/github.com/lrstanley/girc/cap.go generated vendored
View File

@ -64,7 +64,7 @@ func possibleCapList(c *Client) map[string][]string {
if !c.Config.DisableSTS && !c.Config.SSL { if !c.Config.DisableSTS && !c.Config.SSL {
// If fallback supported, and we failed recently, don't try negotiating STS. // If fallback supported, and we failed recently, don't try negotiating STS.
// ONLY do this fallback if we're expired (primarily useful during the first // ONLY do this fallback if we're expired (primarily useful during the first
// sts negotation). // sts negotiation).
if time.Since(c.state.sts.lastFailed) < 5*time.Minute && !c.Config.DisableSTSFallback { if time.Since(c.state.sts.lastFailed) < 5*time.Minute && !c.Config.DisableSTSFallback {
c.debug.Println("skipping strict transport policy negotiation; failed within the last 5 minutes") c.debug.Println("skipping strict transport policy negotiation; failed within the last 5 minutes")
} else { } else {

7
vendor/github.com/lrstanley/girc/client.go generated vendored
View File

@ -10,7 +10,6 @@ import (
"errors" "errors"
"fmt" "fmt"
"io" "io"
"io/ioutil"
"log" "log"
"net" "net"
"os" "os"
@ -268,7 +267,7 @@ func New(config Config) *Client {
if envDebug { if envDebug {
c.debug = log.New(os.Stderr, "debug:", log.Ltime|log.Lshortfile) c.debug = log.New(os.Stderr, "debug:", log.Ltime|log.Lshortfile)
} else { } else {
c.debug = log.New(ioutil.Discard, "", 0) c.debug = log.New(io.Discard, "", 0)
} }
} else { } else {
if envDebug { if envDebug {
@ -411,7 +410,7 @@ func (c *Client) execLoop(ctx context.Context, errs chan error, wg *sync.WaitGro
// Handles incoming ERROR responses. These are only ever sent // Handles incoming ERROR responses. These are only ever sent
// by the server (with the exception that this library may use // by the server (with the exception that this library may use
// them as a lower level way of signalling to disconnect due // them as a lower level way of signalling to disconnect due
// to some other client-choosen error), and should always be // to some other client-chosen error), and should always be
// followed up by the server disconnecting the client. If for // followed up by the server disconnecting the client. If for
// some reason the server doesn't disconnect the client, or // some reason the server doesn't disconnect the client, or
// if this library is the source of the error, this should // if this library is the source of the error, this should
@ -446,7 +445,7 @@ func (c *Client) DisableTracking() {
// Server returns the string representation of host+port pair for the connection. // Server returns the string representation of host+port pair for the connection.
func (c *Client) Server() string { func (c *Client) Server() string {
c.state.Lock() c.state.Lock()
defer c.state.Lock() defer c.state.Unlock()
return c.server() return c.server()
} }

6
vendor/github.com/lrstanley/girc/commands.go generated vendored
View File

@ -5,9 +5,9 @@
package girc package girc
import ( import (
"strconv"
"errors" "errors"
"fmt" "fmt"
"strconv"
) )
// Commands holds a large list of useful methods to interact with the server, // Commands holds a large list of useful methods to interact with the server,
@ -37,7 +37,7 @@ func (cmd *Commands) Join(channels ...string) {
continue continue
} }
if len(buffer) == 0 { if buffer == "" {
buffer = channels[i] buffer = channels[i]
} else { } else {
buffer += "," + channels[i] buffer += "," + channels[i]
@ -341,7 +341,7 @@ func (cmd *Commands) List(channels ...string) {
continue continue
} }
if len(buffer) == 0 { if buffer == "" {
buffer = channels[i] buffer = channels[i]
} else { } else {
buffer += "," + channels[i] buffer += "," + channels[i]

2
vendor/github.com/lrstanley/girc/constants.go generated vendored
View File

@ -149,7 +149,7 @@ const (
RPL_ENDOFWHOWAS = "369" RPL_ENDOFWHOWAS = "369"
RPL_LISTSTART = "321" RPL_LISTSTART = "321"
RPL_LIST = "322" RPL_LIST = "322"
RPL_LISTEND = "323" RPL_LISTEND = "323" //nolint:misspell // it's correct.
RPL_UNIQOPIS = "325" RPL_UNIQOPIS = "325"
RPL_CHANNELMODEIS = "324" RPL_CHANNELMODEIS = "324"
RPL_NOTOPIC = "331" RPL_NOTOPIC = "331"

2
vendor/github.com/lrstanley/girc/ctcp.go generated vendored
View File

@ -104,7 +104,7 @@ func EncodeCTCP(ctcp *CTCPEvent) (out string) {
// EncodeCTCPRaw is much like EncodeCTCP, however accepts a raw command and // EncodeCTCPRaw is much like EncodeCTCP, however accepts a raw command and
// string as input. // string as input.
func EncodeCTCPRaw(cmd, text string) (out string) { func EncodeCTCPRaw(cmd, text string) (out string) {
if len(cmd) <= 0 { if cmd == "" {
return "" return ""
} }

3
vendor/github.com/lrstanley/girc/event.go generated vendored
View File

@ -283,7 +283,6 @@ func (e *Event) Bytes() []byte {
// Space separated list of arguments. // Space separated list of arguments.
if len(e.Params) > 0 { if len(e.Params) > 0 {
// buffer.WriteByte(eventSpace)
for i := 0; i < len(e.Params); i++ { for i := 0; i < len(e.Params); i++ {
if i == len(e.Params)-1 && (strings.Contains(e.Params[i], " ") || strings.HasPrefix(e.Params[i], ":") || e.Params[i] == "") { if i == len(e.Params)-1 && (strings.Contains(e.Params[i], " ") || strings.HasPrefix(e.Params[i], ":") || e.Params[i] == "") {
buffer.WriteString(string(eventSpace) + string(messagePrefix) + e.Params[i]) buffer.WriteString(string(eventSpace) + string(messagePrefix) + e.Params[i])
@ -621,7 +620,7 @@ func (s *Source) IsHostmask() bool {
// IsServer returns true if this source looks like a server name. // IsServer returns true if this source looks like a server name.
func (s *Source) IsServer() bool { func (s *Source) IsServer() bool {
return len(s.Ident) <= 0 && len(s.Host) <= 0 return s.Ident == "" && s.Host == ""
} }
// writeTo is an utility function to write the source to the bytes.Buffer // writeTo is an utility function to write the source to the bytes.Buffer

12
vendor/github.com/lrstanley/girc/format.go generated vendored
View File

@ -127,10 +127,10 @@ func Fmt(text string) string {
// See Fmt() for more information. // See Fmt() for more information.
func TrimFmt(text string) string { func TrimFmt(text string) string {
for color := range fmtColors { for color := range fmtColors {
text = strings.Replace(text, string(fmtOpenChar)+color+string(fmtCloseChar), "", -1) text = strings.ReplaceAll(text, string(fmtOpenChar)+color+string(fmtCloseChar), "")
} }
for code := range fmtCodes { for code := range fmtCodes {
text = strings.Replace(text, string(fmtOpenChar)+code+string(fmtCloseChar), "", -1) text = strings.ReplaceAll(text, string(fmtOpenChar)+code+string(fmtCloseChar), "")
} }
return text return text
@ -138,7 +138,7 @@ func TrimFmt(text string) string {
// This is really the only fastest way of doing this (marginally better than // This is really the only fastest way of doing this (marginally better than
// actually trying to parse it manually.) // actually trying to parse it manually.)
var reStripColor = regexp.MustCompile(`\x03([019]?[0-9](,[019]?[0-9])?)?`) var reStripColor = regexp.MustCompile(`\x03([019]?\d(,[019]?\d)?)?`)
// StripRaw tries to strip all ASCII format codes that are used for IRC. // StripRaw tries to strip all ASCII format codes that are used for IRC.
// Primarily, foreground/background colors, and other control bytes like // Primarily, foreground/background colors, and other control bytes like
@ -148,7 +148,7 @@ func StripRaw(text string) string {
text = reStripColor.ReplaceAllString(text, "") text = reStripColor.ReplaceAllString(text, "")
for _, code := range fmtCodes { for _, code := range fmtCodes {
text = strings.Replace(text, code, "", -1) text = strings.ReplaceAll(text, code, "")
} }
return text return text
@ -219,7 +219,7 @@ func IsValidChannel(channel string) bool {
// digit = 0x30-0x39 // digit = 0x30-0x39
// special = 0x5B-0x60 / 0x7B-0x7D // special = 0x5B-0x60 / 0x7B-0x7D
func IsValidNick(nick string) bool { func IsValidNick(nick string) bool {
if len(nick) <= 0 { if nick == "" {
return false return false
} }
@ -256,7 +256,7 @@ func IsValidNick(nick string) bool {
// user = 1*( %x01-09 / %x0B-0C / %x0E-1F / %x21-3F / %x41-FF ) // user = 1*( %x01-09 / %x0B-0C / %x0E-1F / %x21-3F / %x41-FF )
// ; any octet except NUL, CR, LF, " " and "@" // ; any octet except NUL, CR, LF, " " and "@"
func IsValidUser(name string) bool { func IsValidUser(name string) bool {
if len(name) <= 0 { if name == "" {
return false return false
} }

2
vendor/github.com/lrstanley/girc/handler.go generated vendored
View File

@ -287,7 +287,7 @@ func (c *Caller) Remove(cuid string) (success bool) {
// on your own. // on your own.
func (c *Caller) remove(cuid string) (success bool) { func (c *Caller) remove(cuid string) (success bool) {
cmd, uid := c.cuidToID(cuid) cmd, uid := c.cuidToID(cuid)
if len(cmd) == 0 || len(uid) == 0 { if cmd == "" || uid == "" {
return false return false
} }

26
vendor/github.com/lrstanley/girc/modes.go generated vendored
View File

@ -34,7 +34,7 @@ func (c *CMode) Short() string {
// String returns a string representation of a mode, including optional // String returns a string representation of a mode, including optional
// arguments. E.g. "+b user*!ident@host.*.com" // arguments. E.g. "+b user*!ident@host.*.com"
func (c *CMode) String() string { func (c *CMode) String() string {
if len(c.args) == 0 { if c.args == "" {
return c.Short() return c.Short()
} }
@ -106,7 +106,7 @@ func (c *CModes) HasMode(mode string) bool {
func (c *CModes) Get(mode string) (args string, ok bool) { func (c *CModes) Get(mode string) (args string, ok bool) {
for i := 0; i < len(c.modes); i++ { for i := 0; i < len(c.modes); i++ {
if string(c.modes[i].name) == mode { if string(c.modes[i].name) == mode {
if len(c.modes[i].args) == 0 { if c.modes[i].args == "" {
return "", false return "", false
} }
@ -157,7 +157,7 @@ func (c *CModes) hasArg(set bool, mode byte) (hasArgs, isSetting bool) {
// For example, the latter would mean applying an incoming MODE with the modes // For example, the latter would mean applying an incoming MODE with the modes
// stored for a channel. // stored for a channel.
func (c *CModes) Apply(modes []CMode) { func (c *CModes) Apply(modes []CMode) {
var new []CMode var newModes []CMode
for j := 0; j < len(c.modes); j++ { for j := 0; j < len(c.modes); j++ {
isin := false isin := false
@ -166,14 +166,14 @@ func (c *CModes) Apply(modes []CMode) {
continue continue
} }
if c.modes[j].name == modes[i].name && modes[i].add { if c.modes[j].name == modes[i].name && modes[i].add {
new = append(new, modes[i]) newModes = append(newModes, modes[i])
isin = true isin = true
break break
} }
} }
if !isin { if !isin {
new = append(new, c.modes[j]) newModes = append(newModes, c.modes[j])
} }
} }
@ -183,19 +183,19 @@ func (c *CModes) Apply(modes []CMode) {
} }
isin := false isin := false
for j := 0; j < len(new); j++ { for j := 0; j < len(newModes); j++ {
if modes[i].name == new[j].name { if modes[i].name == newModes[j].name {
isin = true isin = true
break break
} }
} }
if !isin { if !isin {
new = append(new, modes[i]) newModes = append(newModes, modes[i])
} }
} }
c.modes = new c.modes = newModes
} }
// Parse parses a set of flags and args, returning the necessary list of // Parse parses a set of flags and args, returning the necessary list of
@ -221,7 +221,7 @@ func (c *CModes) Parse(flags string, args []string) (out []CMode) {
} }
hasArgs, isSetting := c.hasArg(add, flags[i]) hasArgs, isSetting := c.hasArg(add, flags[i])
if hasArgs && len(args) >= argCount+1 { if hasArgs && len(args) > argCount {
mode.args = args[argCount] mode.args = args[argCount]
argCount++ argCount++
} }
@ -351,7 +351,7 @@ func handleMODE(c *Client, e Event) {
// Loop through and update users modes as necessary. // Loop through and update users modes as necessary.
for i := 0; i < len(modes); i++ { for i := 0; i < len(modes); i++ {
if modes[i].setting || len(modes[i].args) == 0 { if modes[i].setting || modes[i].args == "" {
continue continue
} }
@ -493,8 +493,8 @@ func (m *Perms) reset() {
// set translates raw prefix characters into proper permissions. Only // set translates raw prefix characters into proper permissions. Only
// use this function when you have a session lock. // use this function when you have a session lock.
func (m *Perms) set(prefix string, append bool) { func (m *Perms) set(prefix string, add bool) {
if !append { if !add {
m.reset() m.reset()
} }

11
vendor/google.golang.org/protobuf/encoding/protowire/wire.go generated vendored
View File

@ -25,6 +25,7 @@ const (
FirstReservedNumber Number = 19000 FirstReservedNumber Number = 19000
LastReservedNumber Number = 19999 LastReservedNumber Number = 19999
MaxValidNumber Number = 1<<29 - 1 MaxValidNumber Number = 1<<29 - 1
DefaultRecursionLimit = 10000
) )
// IsValid reports whether the field number is semantically valid. // IsValid reports whether the field number is semantically valid.
@ -55,6 +56,7 @@ const (
errCodeOverflow errCodeOverflow
errCodeReserved errCodeReserved
errCodeEndGroup errCodeEndGroup
errCodeRecursionDepth
) )
var ( var (
@ -112,6 +114,10 @@ func ConsumeField(b []byte) (Number, Type, int) {
// When parsing a group, the length includes the end group marker and // When parsing a group, the length includes the end group marker and
// the end group is verified to match the starting field number. // the end group is verified to match the starting field number.
func ConsumeFieldValue(num Number, typ Type, b []byte) (n int) { func ConsumeFieldValue(num Number, typ Type, b []byte) (n int) {
return consumeFieldValueD(num, typ, b, DefaultRecursionLimit)
}
func consumeFieldValueD(num Number, typ Type, b []byte, depth int) (n int) {
switch typ { switch typ {
case VarintType: case VarintType:
_, n = ConsumeVarint(b) _, n = ConsumeVarint(b)
@ -126,6 +132,9 @@ func ConsumeFieldValue(num Number, typ Type, b []byte) (n int) {
_, n = ConsumeBytes(b) _, n = ConsumeBytes(b)
return n return n
case StartGroupType: case StartGroupType:
if depth < 0 {
return errCodeRecursionDepth
}
n0 := len(b) n0 := len(b)
for { for {
num2, typ2, n := ConsumeTag(b) num2, typ2, n := ConsumeTag(b)
@ -140,7 +149,7 @@ func ConsumeFieldValue(num Number, typ Type, b []byte) (n int) {
return n0 - len(b) return n0 - len(b)
} }
n = ConsumeFieldValue(num2, typ2, b) n = consumeFieldValueD(num2, typ2, b, depth-1)
if n < 0 { if n < 0 {
return n // forward error code return n // forward error code
} }

2
vendor/google.golang.org/protobuf/internal/encoding/text/decode.go generated vendored
View File

@ -381,7 +381,7 @@ func (d *Decoder) currentOpenKind() (Kind, byte) {
case '[': case '[':
return ListOpen, ']' return ListOpen, ']'
} }
panic(fmt.Sprintf("Decoder: openStack contains invalid byte %s", string(openCh))) panic(fmt.Sprintf("Decoder: openStack contains invalid byte %c", openCh))
} }
func (d *Decoder) pushOpenStack(ch byte) { func (d *Decoder) pushOpenStack(ch byte) {

1
vendor/google.golang.org/protobuf/internal/errors/is_go112.go generated vendored
View File

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style // Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file. // license that can be found in the LICENSE file.
//go:build !go1.13
// +build !go1.13 // +build !go1.13
package errors package errors

1
vendor/google.golang.org/protobuf/internal/errors/is_go113.go generated vendored
View File

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style // Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file. // license that can be found in the LICENSE file.
//go:build go1.13
// +build go1.13 // +build go1.13
package errors package errors

1
vendor/google.golang.org/protobuf/internal/flags/proto_legacy_disable.go generated vendored
View File

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style // Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file. // license that can be found in the LICENSE file.
//go:build !protolegacy
// +build !protolegacy // +build !protolegacy
package flags package flags

1
vendor/google.golang.org/protobuf/internal/flags/proto_legacy_enable.go generated vendored
View File

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style // Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file. // license that can be found in the LICENSE file.
//go:build protolegacy
// +build protolegacy // +build protolegacy
package flags package flags

1
vendor/google.golang.org/protobuf/internal/impl/codec_map_go111.go generated vendored
View File

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style // Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file. // license that can be found in the LICENSE file.
//go:build !go1.12
// +build !go1.12 // +build !go1.12
package impl package impl

1
vendor/google.golang.org/protobuf/internal/impl/codec_map_go112.go generated vendored
View File

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style // Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file. // license that can be found in the LICENSE file.
//go:build go1.12
// +build go1.12 // +build go1.12
package impl package impl

1
vendor/google.golang.org/protobuf/internal/impl/codec_reflect.go generated vendored
View File

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style // Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file. // license that can be found in the LICENSE file.
//go:build purego || appengine
// +build purego appengine // +build purego appengine
package impl package impl

1
vendor/google.golang.org/protobuf/internal/impl/codec_unsafe.go generated vendored
View File

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style // Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file. // license that can be found in the LICENSE file.
//go:build !purego && !appengine
// +build !purego,!appengine // +build !purego,!appengine
package impl package impl

8
vendor/google.golang.org/protobuf/internal/impl/decode.go generated vendored
View File

@ -18,6 +18,7 @@ import (
) )
var errDecode = errors.New("cannot parse invalid wire-format data") var errDecode = errors.New("cannot parse invalid wire-format data")
var errRecursionDepth = errors.New("exceeded maximum recursion depth")
type unmarshalOptions struct { type unmarshalOptions struct {
flags protoiface.UnmarshalInputFlags flags protoiface.UnmarshalInputFlags
@ -25,6 +26,7 @@ type unmarshalOptions struct {
FindExtensionByName(field protoreflect.FullName) (protoreflect.ExtensionType, error) FindExtensionByName(field protoreflect.FullName) (protoreflect.ExtensionType, error)
FindExtensionByNumber(message protoreflect.FullName, field protoreflect.FieldNumber) (protoreflect.ExtensionType, error) FindExtensionByNumber(message protoreflect.FullName, field protoreflect.FieldNumber) (protoreflect.ExtensionType, error)
} }
depth int
} }
func (o unmarshalOptions) Options() proto.UnmarshalOptions { func (o unmarshalOptions) Options() proto.UnmarshalOptions {
@ -44,6 +46,7 @@ func (o unmarshalOptions) IsDefault() bool {
var lazyUnmarshalOptions = unmarshalOptions{ var lazyUnmarshalOptions = unmarshalOptions{
resolver: preg.GlobalTypes, resolver: preg.GlobalTypes,
depth: protowire.DefaultRecursionLimit,
} }
type unmarshalOutput struct { type unmarshalOutput struct {
@ -62,6 +65,7 @@ func (mi *MessageInfo) unmarshal(in piface.UnmarshalInput) (piface.UnmarshalOutp
out, err := mi.unmarshalPointer(in.Buf, p, 0, unmarshalOptions{ out, err := mi.unmarshalPointer(in.Buf, p, 0, unmarshalOptions{
flags: in.Flags, flags: in.Flags,
resolver: in.Resolver, resolver: in.Resolver,
depth: in.Depth,
}) })
var flags piface.UnmarshalOutputFlags var flags piface.UnmarshalOutputFlags
if out.initialized { if out.initialized {
@ -82,6 +86,10 @@ var errUnknown = errors.New("unknown")
func (mi *MessageInfo) unmarshalPointer(b []byte, p pointer, groupTag protowire.Number, opts unmarshalOptions) (out unmarshalOutput, err error) { func (mi *MessageInfo) unmarshalPointer(b []byte, p pointer, groupTag protowire.Number, opts unmarshalOptions) (out unmarshalOutput, err error) {
mi.init() mi.init()
opts.depth--
if opts.depth < 0 {
return out, errRecursionDepth
}
if flags.ProtoLegacy && mi.isMessageSet { if flags.ProtoLegacy && mi.isMessageSet {
return unmarshalMessageSet(mi, b, p, opts) return unmarshalMessageSet(mi, b, p, opts)
} }

1
vendor/google.golang.org/protobuf/internal/impl/pointer_reflect.go generated vendored
View File

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style // Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file. // license that can be found in the LICENSE file.
//go:build purego || appengine
// +build purego appengine // +build purego appengine
package impl package impl

1
vendor/google.golang.org/protobuf/internal/impl/pointer_unsafe.go generated vendored
View File

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style // Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file. // license that can be found in the LICENSE file.
//go:build !purego && !appengine
// +build !purego,!appengine // +build !purego,!appengine
package impl package impl

1
vendor/google.golang.org/protobuf/internal/strs/strings_pure.go generated vendored
View File

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style // Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file. // license that can be found in the LICENSE file.
//go:build purego || appengine
// +build purego appengine // +build purego appengine
package strs package strs

1
vendor/google.golang.org/protobuf/internal/strs/strings_unsafe.go generated vendored
View File

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style // Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file. // license that can be found in the LICENSE file.
//go:build !purego && !appengine
// +build !purego,!appengine // +build !purego,!appengine
package strs package strs

4
vendor/google.golang.org/protobuf/internal/version/version.go generated vendored
View File

@ -52,8 +52,8 @@ import (
// 10. Send out the CL for review and submit it. // 10. Send out the CL for review and submit it.
const ( const (
Major = 1 Major = 1
Minor = 27 Minor = 28
Patch = 1 Patch = 0
PreRelease = "" PreRelease = ""
) )

17
vendor/google.golang.org/protobuf/proto/decode.go generated vendored
View File

@ -42,18 +42,25 @@ type UnmarshalOptions struct {
FindExtensionByName(field protoreflect.FullName) (protoreflect.ExtensionType, error) FindExtensionByName(field protoreflect.FullName) (protoreflect.ExtensionType, error)
FindExtensionByNumber(message protoreflect.FullName, field protoreflect.FieldNumber) (protoreflect.ExtensionType, error) FindExtensionByNumber(message protoreflect.FullName, field protoreflect.FieldNumber) (protoreflect.ExtensionType, error)
} }
// RecursionLimit limits how deeply messages may be nested.
// If zero, a default limit is applied.
RecursionLimit int
} }
// Unmarshal parses the wire-format message in b and places the result in m. // Unmarshal parses the wire-format message in b and places the result in m.
// The provided message must be mutable (e.g., a non-nil pointer to a message). // The provided message must be mutable (e.g., a non-nil pointer to a message).
func Unmarshal(b []byte, m Message) error { func Unmarshal(b []byte, m Message) error {
_, err := UnmarshalOptions{}.unmarshal(b, m.ProtoReflect()) _, err := UnmarshalOptions{RecursionLimit: protowire.DefaultRecursionLimit}.unmarshal(b, m.ProtoReflect())
return err return err
} }
// Unmarshal parses the wire-format message in b and places the result in m. // Unmarshal parses the wire-format message in b and places the result in m.
// The provided message must be mutable (e.g., a non-nil pointer to a message). // The provided message must be mutable (e.g., a non-nil pointer to a message).
func (o UnmarshalOptions) Unmarshal(b []byte, m Message) error { func (o UnmarshalOptions) Unmarshal(b []byte, m Message) error {
if o.RecursionLimit == 0 {
o.RecursionLimit = protowire.DefaultRecursionLimit
}
_, err := o.unmarshal(b, m.ProtoReflect()) _, err := o.unmarshal(b, m.ProtoReflect())
return err return err
} }
@ -63,6 +70,9 @@ func (o UnmarshalOptions) Unmarshal(b []byte, m Message) error {
// This method permits fine-grained control over the unmarshaler. // This method permits fine-grained control over the unmarshaler.
// Most users should use Unmarshal instead. // Most users should use Unmarshal instead.
func (o UnmarshalOptions) UnmarshalState(in protoiface.UnmarshalInput) (protoiface.UnmarshalOutput, error) { func (o UnmarshalOptions) UnmarshalState(in protoiface.UnmarshalInput) (protoiface.UnmarshalOutput, error) {
if o.RecursionLimit == 0 {
o.RecursionLimit = protowire.DefaultRecursionLimit
}
return o.unmarshal(in.Buf, in.Message) return o.unmarshal(in.Buf, in.Message)
} }
@ -86,12 +96,17 @@ func (o UnmarshalOptions) unmarshal(b []byte, m protoreflect.Message) (out proto
Message: m, Message: m,
Buf: b, Buf: b,
Resolver: o.Resolver, Resolver: o.Resolver,
Depth: o.RecursionLimit,
} }
if o.DiscardUnknown { if o.DiscardUnknown {
in.Flags |= protoiface.UnmarshalDiscardUnknown in.Flags |= protoiface.UnmarshalDiscardUnknown
} }
out, err = methods.Unmarshal(in) out, err = methods.Unmarshal(in)
} else { } else {
o.RecursionLimit--
if o.RecursionLimit < 0 {
return out, errors.New("exceeded max recursion depth")
}
err = o.unmarshalMessageSlow(b, m) err = o.unmarshalMessageSlow(b, m)
} }
if err != nil { if err != nil {

1
vendor/google.golang.org/protobuf/proto/proto_methods.go generated vendored
View File

@ -3,6 +3,7 @@
// license that can be found in the LICENSE file. // license that can be found in the LICENSE file.
// The protoreflect build tag disables use of fast-path methods. // The protoreflect build tag disables use of fast-path methods.
//go:build !protoreflect
// +build !protoreflect // +build !protoreflect
package proto package proto

1
vendor/google.golang.org/protobuf/proto/proto_reflect.go generated vendored
View File

@ -3,6 +3,7 @@
// license that can be found in the LICENSE file. // license that can be found in the LICENSE file.
// The protoreflect build tag disables use of fast-path methods. // The protoreflect build tag disables use of fast-path methods.
//go:build protoreflect
// +build protoreflect // +build protoreflect
package proto package proto

1
vendor/google.golang.org/protobuf/reflect/protoreflect/methods.go generated vendored
View File

@ -53,6 +53,7 @@ type (
FindExtensionByName(field FullName) (ExtensionType, error) FindExtensionByName(field FullName) (ExtensionType, error)
FindExtensionByNumber(message FullName, field FieldNumber) (ExtensionType, error) FindExtensionByNumber(message FullName, field FieldNumber) (ExtensionType, error)
} }
Depth int
} }
unmarshalOutput = struct { unmarshalOutput = struct {
pragma.NoUnkeyedLiterals pragma.NoUnkeyedLiterals

1
vendor/google.golang.org/protobuf/reflect/protoreflect/value_pure.go generated vendored
View File

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style // Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file. // license that can be found in the LICENSE file.
//go:build purego || appengine
// +build purego appengine // +build purego appengine
package protoreflect package protoreflect

25
vendor/google.golang.org/protobuf/reflect/protoreflect/value_union.go generated vendored
View File

@ -41,6 +41,31 @@ import (
// Converting to/from a Value and a concrete Go value panics on type mismatch. // Converting to/from a Value and a concrete Go value panics on type mismatch.
// For example, ValueOf("hello").Int() panics because this attempts to // For example, ValueOf("hello").Int() panics because this attempts to
// retrieve an int64 from a string. // retrieve an int64 from a string.
//
// List, Map, and Message Values are called "composite" values.
//
// A composite Value may alias (reference) memory at some location,
// such that changes to the Value updates the that location.
// A composite value acquired with a Mutable method, such as Message.Mutable,
// always references the source object.
//
// For example:
// // Append a 0 to a "repeated int32" field.
// // Since the Value returned by Mutable is guaranteed to alias
// // the source message, modifying the Value modifies the message.
// message.Mutable(fieldDesc).(List).Append(protoreflect.ValueOfInt32(0))
//
// // Assign [0] to a "repeated int32" field by creating a new Value,
// // modifying it, and assigning it.
// list := message.NewField(fieldDesc).(List)
// list.Append(protoreflect.ValueOfInt32(0))
// message.Set(fieldDesc, list)
// // ERROR: Since it is not defined whether Set aliases the source,
// // appending to the List here may or may not modify the message.
// list.Append(protoreflect.ValueOfInt32(0))
//
// Some operations, such as Message.Get, may return an "empty, read-only"
// composite Value. Modifying an empty, read-only value panics.
type Value value type Value value
// The protoreflect API uses a custom Value union type instead of interface{} // The protoreflect API uses a custom Value union type instead of interface{}

1
vendor/google.golang.org/protobuf/reflect/protoreflect/value_unsafe.go generated vendored
View File

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style // Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file. // license that can be found in the LICENSE file.
//go:build !purego && !appengine
// +build !purego,!appengine // +build !purego,!appengine
package protoreflect package protoreflect

1
vendor/google.golang.org/protobuf/runtime/protoiface/methods.go generated vendored
View File

@ -103,6 +103,7 @@ type UnmarshalInput = struct {
FindExtensionByName(field protoreflect.FullName) (protoreflect.ExtensionType, error) FindExtensionByName(field protoreflect.FullName) (protoreflect.ExtensionType, error)
FindExtensionByNumber(message protoreflect.FullName, field protoreflect.FieldNumber) (protoreflect.ExtensionType, error) FindExtensionByNumber(message protoreflect.FullName, field protoreflect.FieldNumber) (protoreflect.ExtensionType, error)
} }
Depth int
} }
// UnmarshalOutput is output from the Unmarshal method. // UnmarshalOutput is output from the Unmarshal method.

1
vendor/modernc.org/sqlite/AUTHORS generated vendored
View File

@ -13,6 +13,7 @@ Davsk Ltd Co <skinner.david@gmail.com>
Jaap Aarts <jaap.aarts1@gmail.com> Jaap Aarts <jaap.aarts1@gmail.com>
Jan Mercl <0xjnml@gmail.com> Jan Mercl <0xjnml@gmail.com>
Logan Snow <logansnow@protonmail.com> Logan Snow <logansnow@protonmail.com>
Michael Hoffmann <mhoffm@posteo.de>
Ross Light <ross@zombiezen.com> Ross Light <ross@zombiezen.com>
Steffen Butzer <steffen(dot)butzer@outlook.com> Steffen Butzer <steffen(dot)butzer@outlook.com>
Saed SayedAhmed <saadmtsa@gmail.com> Saed SayedAhmed <saadmtsa@gmail.com>

1
vendor/modernc.org/sqlite/CONTRIBUTORS generated vendored
View File

@ -14,6 +14,7 @@ Jaap Aarts <jaap.aarts1@gmail.com>
Jan Mercl <0xjnml@gmail.com> Jan Mercl <0xjnml@gmail.com>
Logan Snow <logansnow@protonmail.com> Logan Snow <logansnow@protonmail.com>
Matthew Gabeler-Lee <fastcat@gmail.com> Matthew Gabeler-Lee <fastcat@gmail.com>
Michael Hoffmann <mhoffm@posteo.de>
Ross Light <ross@zombiezen.com> Ross Light <ross@zombiezen.com>
Steffen Butzer <steffen(dot)butzer@outlook.com> Steffen Butzer <steffen(dot)butzer@outlook.com>
Yaacov Akiba Slama <ya@slamail.org> Yaacov Akiba Slama <ya@slamail.org>

6
vendor/modernc.org/sqlite/doc.go generated vendored
View File

@ -37,6 +37,12 @@
// //
// Changelog // Changelog
// //
// 2022-04-04 v1.16.0:
//
// Support scalar application defined functions written in Go.
//
// https://www.sqlite.org/appfunc.html
//
// 2022-03-13 v1.15.0: // 2022-03-13 v1.15.0:
// //
// Support linux/riscv64. // Support linux/riscv64.

10
vendor/modernc.org/sqlite/lib/defs.go generated vendored Normal file
View File

@ -0,0 +1,10 @@
// Copyright 2022 The Sqlite Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package sqlite3
const (
SQLITE_STATIC = uintptr(0) // ((sqlite3_destructor_type)0)
SQLITE_TRANSIENT = ^uintptr(0) // ((sqlite3_destructor_type)-1)
)

223
vendor/modernc.org/sqlite/sqlite.go generated vendored
View File

@ -1090,14 +1090,18 @@ func (c *conn) bindText(pstmt uintptr, idx1 int, value string) (uintptr, error)
// int sqlite3_bind_blob(sqlite3_stmt*, int, const void*, int n, void(*)(void*)); // int sqlite3_bind_blob(sqlite3_stmt*, int, const void*, int n, void(*)(void*));
func (c *conn) bindBlob(pstmt uintptr, idx1 int, value []byte) (uintptr, error) { func (c *conn) bindBlob(pstmt uintptr, idx1 int, value []byte) (uintptr, error) {
if len(value) == 0 {
if rc := sqlite3.Xsqlite3_bind_zeroblob(c.tls, pstmt, int32(idx1), 0); rc != sqlite3.SQLITE_OK {
return 0, c.errstr(rc)
}
return 0, nil
}
p, err := c.malloc(len(value)) p, err := c.malloc(len(value))
if err != nil { if err != nil {
return 0, err return 0, err
} }
if len(value) != 0 {
copy((*libc.RawMem)(unsafe.Pointer(p))[:len(value):len(value)], value) copy((*libc.RawMem)(unsafe.Pointer(p))[:len(value):len(value)], value)
}
if rc := sqlite3.Xsqlite3_bind_blob(c.tls, pstmt, int32(idx1), p, int32(len(value)), 0); rc != sqlite3.SQLITE_OK { if rc := sqlite3.Xsqlite3_bind_blob(c.tls, pstmt, int32(idx1), p, int32(len(value)), 0); rc != sqlite3.SQLITE_OK {
c.free(p) c.free(p)
return 0, c.errstr(rc) return 0, c.errstr(rc)
@ -1307,6 +1311,7 @@ func (c *conn) Close() error {
c.db = 0 c.db = 0
} }
if c.tls != nil { if c.tls != nil {
c.tls.Close() c.tls.Close()
c.tls = nil c.tls = nil
@ -1323,6 +1328,32 @@ func (c *conn) closeV2(db uintptr) error {
return nil return nil
} }
type userDefinedFunction struct {
zFuncName uintptr
nArg int32
eTextRep int32
xFunc func(*libc.TLS, uintptr, int32, uintptr)
freeOnce sync.Once
}
func (c *conn) createFunctionInternal(fun *userDefinedFunction) error {
if rc := sqlite3.Xsqlite3_create_function(
c.tls,
c.db,
fun.zFuncName,
fun.nArg,
fun.eTextRep,
0,
*(*uintptr)(unsafe.Pointer(&fun.xFunc)),
0,
0,
); rc != sqlite3.SQLITE_OK {
return c.errstr(rc)
}
return nil
}
// Execer is an optional interface that may be implemented by a Conn. // Execer is an optional interface that may be implemented by a Conn.
// //
// If a Conn does not implement Execer, the sql package's DB.Exec will first // If a Conn does not implement Execer, the sql package's DB.Exec will first
@ -1388,9 +1419,14 @@ func (c *conn) query(ctx context.Context, query string, args []driver.NamedValue
} }
// Driver implements database/sql/driver.Driver. // Driver implements database/sql/driver.Driver.
type Driver struct{} type Driver struct {
// user defined functions that are added to every new connection on Open
udfs map[string]*userDefinedFunction
}
func newDriver() *Driver { return &Driver{} } var d = &Driver{udfs: make(map[string]*userDefinedFunction)}
func newDriver() *Driver { return d }
// Open returns a new connection to the database. The name is a string in a // Open returns a new connection to the database. The name is a string in a
// driver-specific format. // driver-specific format.
@ -1422,5 +1458,180 @@ func newDriver() *Driver { return &Driver{} }
// available at // available at
// https://www.sqlite.org/lang_transaction.html#deferred_immediate_and_exclusive_transactions // https://www.sqlite.org/lang_transaction.html#deferred_immediate_and_exclusive_transactions
func (d *Driver) Open(name string) (driver.Conn, error) { func (d *Driver) Open(name string) (driver.Conn, error) {
return newConn(name) c, err := newConn(name)
if err != nil {
return nil, err
}
for _, udf := range d.udfs {
if err = c.createFunctionInternal(udf); err != nil {
c.Close()
return nil, err
}
}
return c, nil
}
// FunctionContext represents the context user defined functions execute in.
// Fields and/or methods of this type may get addedd in the future.
type FunctionContext struct{}
const sqliteValPtrSize = unsafe.Sizeof(&sqlite3.Sqlite3_value{})
// RegisterScalarFunction registers a scalar function named zFuncName with nArg
// arguments. Passing -1 for nArg indicates the function is variadic.
//
// The new function will be available to all new connections opened after
// executing RegisterScalarFunction.
func RegisterScalarFunction(
zFuncName string,
nArg int32,
xFunc func(ctx *FunctionContext, args []driver.Value) (driver.Value, error),
) error {
return registerScalarFunction(zFuncName, nArg, sqlite3.SQLITE_UTF8, xFunc)
}
// MustRegisterScalarFunction is like RegisterScalarFunction but panics on
// error.
func MustRegisterScalarFunction(
zFuncName string,
nArg int32,
xFunc func(ctx *FunctionContext, args []driver.Value) (driver.Value, error),
) {
if err := RegisterScalarFunction(zFuncName, nArg, xFunc); err != nil {
panic(err)
}
}
// MustRegisterDeterministicScalarFunction is like
// RegisterDeterministicScalarFunction but panics on error.
func MustRegisterDeterministicScalarFunction(
zFuncName string,
nArg int32,
xFunc func(ctx *FunctionContext, args []driver.Value) (driver.Value, error),
) {
if err := RegisterDeterministicScalarFunction(zFuncName, nArg, xFunc); err != nil {
panic(err)
}
}
// RegisterDeterministicScalarFunction registers a deterministic scalar
// function named zFuncName with nArg arguments. Passing -1 for nArg indicates
// the function is variadic. A deterministic function means that the function
// always gives the same output when the input parameters are the same.
//
// The new function will be available to all new connections opened after
// executing RegisterDeterministicScalarFunction.
func RegisterDeterministicScalarFunction(
zFuncName string,
nArg int32,
xFunc func(ctx *FunctionContext, args []driver.Value) (driver.Value, error),
) error {
return registerScalarFunction(zFuncName, nArg, sqlite3.SQLITE_UTF8|sqlite3.SQLITE_DETERMINISTIC, xFunc)
}
func registerScalarFunction(
zFuncName string,
nArg int32,
eTextRep int32,
xFunc func(ctx *FunctionContext, args []driver.Value) (driver.Value, error),
) error {
if _, ok := d.udfs[zFuncName]; ok {
return fmt.Errorf("a function named %q is already registered", zFuncName)
}
// dont free, functions registered on the driver live as long as the program
name, err := libc.CString(zFuncName)
if err != nil {
return err
}
udf := &userDefinedFunction{
zFuncName: name,
nArg: nArg,
eTextRep: eTextRep,
xFunc: func(tls *libc.TLS, ctx uintptr, argc int32, argv uintptr) {
setErrorResult := func(res error) {
errmsg, cerr := libc.CString(res.Error())
if cerr != nil {
panic(cerr)
}
defer libc.Xfree(tls, errmsg)
sqlite3.Xsqlite3_result_error(tls, ctx, errmsg, -1)
sqlite3.Xsqlite3_result_error_code(tls, ctx, sqlite3.SQLITE_ERROR)
}
args := make([]driver.Value, argc)
for i := int32(0); i < argc; i++ {
valPtr := *(*uintptr)(unsafe.Pointer(argv + uintptr(i)*sqliteValPtrSize))
switch valType := sqlite3.Xsqlite3_value_type(tls, valPtr); valType {
case sqlite3.SQLITE_TEXT:
args[i] = libc.GoString(sqlite3.Xsqlite3_value_text(tls, valPtr))
case sqlite3.SQLITE_INTEGER:
args[i] = sqlite3.Xsqlite3_value_int64(tls, valPtr)
case sqlite3.SQLITE_FLOAT:
args[i] = sqlite3.Xsqlite3_value_double(tls, valPtr)
case sqlite3.SQLITE_NULL:
args[i] = nil
case sqlite3.SQLITE_BLOB:
size := sqlite3.Xsqlite3_value_bytes(tls, valPtr)
blobPtr := sqlite3.Xsqlite3_value_blob(tls, valPtr)
v := make([]byte, size)
copy(v, (*libc.RawMem)(unsafe.Pointer(blobPtr))[:size:size])
args[i] = v
default:
panic(fmt.Sprintf("unexpected argument type %q passed by sqlite", valType))
}
}
res, err := xFunc(&FunctionContext{}, args)
if err != nil {
setErrorResult(err)
return
}
switch resTyped := res.(type) {
case nil:
sqlite3.Xsqlite3_result_null(tls, ctx)
case int64:
sqlite3.Xsqlite3_result_int64(tls, ctx, resTyped)
case float64:
sqlite3.Xsqlite3_result_double(tls, ctx, resTyped)
case bool:
sqlite3.Xsqlite3_result_int(tls, ctx, libc.Bool32(resTyped))
case time.Time:
sqlite3.Xsqlite3_result_int64(tls, ctx, resTyped.Unix())
case string:
size := int32(len(resTyped))
cstr, err := libc.CString(resTyped)
if err != nil {
panic(err)
}
defer libc.Xfree(tls, cstr)
sqlite3.Xsqlite3_result_text(tls, ctx, cstr, size, sqlite3.SQLITE_TRANSIENT)
case []byte:
size := int32(len(resTyped))
if size == 0 {
sqlite3.Xsqlite3_result_zeroblob(tls, ctx, 0)
return
}
p := libc.Xmalloc(tls, types.Size_t(size))
if p == 0 {
panic(fmt.Sprintf("unable to allocate space for blob: %d", size))
}
defer libc.Xfree(tls, p)
copy((*libc.RawMem)(unsafe.Pointer(p))[:size:size], resTyped)
sqlite3.Xsqlite3_result_blob(tls, ctx, p, size, sqlite3.SQLITE_TRANSIENT)
default:
setErrorResult(fmt.Errorf("function did not return a valid driver.Value: %T", resTyped))
return
}
},
}
d.udfs[zFuncName] = udf
return nil
} }

14
vendor/modules.txt vendored
View File

@ -38,7 +38,7 @@ github.com/Rhymen/go-whatsapp/binary/token
github.com/Rhymen/go-whatsapp/crypto/cbc github.com/Rhymen/go-whatsapp/crypto/cbc
github.com/Rhymen/go-whatsapp/crypto/curve25519 github.com/Rhymen/go-whatsapp/crypto/curve25519
github.com/Rhymen/go-whatsapp/crypto/hkdf github.com/Rhymen/go-whatsapp/crypto/hkdf
# github.com/SevereCloud/vksdk/v2 v2.13.1 # github.com/SevereCloud/vksdk/v2 v2.14.0
## explicit; go 1.16 ## explicit; go 1.16
github.com/SevereCloud/vksdk/v2 github.com/SevereCloud/vksdk/v2
github.com/SevereCloud/vksdk/v2/api github.com/SevereCloud/vksdk/v2/api
@ -182,7 +182,7 @@ github.com/keybase/go-keybase-chat-bot/kbchat/types/chat1
github.com/keybase/go-keybase-chat-bot/kbchat/types/gregor1 github.com/keybase/go-keybase-chat-bot/kbchat/types/gregor1
github.com/keybase/go-keybase-chat-bot/kbchat/types/keybase1 github.com/keybase/go-keybase-chat-bot/kbchat/types/keybase1
github.com/keybase/go-keybase-chat-bot/kbchat/types/stellar1 github.com/keybase/go-keybase-chat-bot/kbchat/types/stellar1
# github.com/klauspost/compress v1.14.2 # github.com/klauspost/compress v1.15.1
## explicit; go 1.15 ## explicit; go 1.15
github.com/klauspost/compress github.com/klauspost/compress
github.com/klauspost/compress/fse github.com/klauspost/compress/fse
@ -207,7 +207,7 @@ github.com/labstack/gommon/bytes
github.com/labstack/gommon/color github.com/labstack/gommon/color
github.com/labstack/gommon/log github.com/labstack/gommon/log
github.com/labstack/gommon/random github.com/labstack/gommon/random
# github.com/lrstanley/girc v0.0.0-20220321215535-9664730c7858 # github.com/lrstanley/girc v0.0.0-20220409202343-de3f963fb827
## explicit; go 1.12 ## explicit; go 1.12
github.com/lrstanley/girc github.com/lrstanley/girc
# github.com/magiconair/properties v1.8.5 # github.com/magiconair/properties v1.8.5
@ -557,7 +557,7 @@ golang.org/x/crypto/scrypt
golang.org/x/crypto/ssh golang.org/x/crypto/ssh
golang.org/x/crypto/ssh/internal/bcrypt_pbkdf golang.org/x/crypto/ssh/internal/bcrypt_pbkdf
golang.org/x/crypto/ssh/terminal golang.org/x/crypto/ssh/terminal
# golang.org/x/image v0.0.0-20220302094943-723b81ca9867 # golang.org/x/image v0.0.0-20220321031419-a8550c1d254a
## explicit; go 1.12 ## explicit; go 1.12
golang.org/x/image/riff golang.org/x/image/riff
golang.org/x/image/vp8 golang.org/x/image/vp8
@ -654,8 +654,8 @@ google.golang.org/appengine/internal/log
google.golang.org/appengine/internal/remote_api google.golang.org/appengine/internal/remote_api
google.golang.org/appengine/internal/urlfetch google.golang.org/appengine/internal/urlfetch
google.golang.org/appengine/urlfetch google.golang.org/appengine/urlfetch
# google.golang.org/protobuf v1.27.1 # google.golang.org/protobuf v1.28.0
## explicit; go 1.9 ## explicit; go 1.11
google.golang.org/protobuf/encoding/prototext google.golang.org/protobuf/encoding/prototext
google.golang.org/protobuf/encoding/protowire google.golang.org/protobuf/encoding/protowire
google.golang.org/protobuf/internal/descfmt google.golang.org/protobuf/internal/descfmt
@ -750,7 +750,7 @@ modernc.org/memory
# modernc.org/opt v0.1.1 # modernc.org/opt v0.1.1
## explicit; go 1.13 ## explicit; go 1.13
modernc.org/opt modernc.org/opt
# modernc.org/sqlite v1.15.4 # modernc.org/sqlite v1.16.0
## explicit; go 1.16 ## explicit; go 1.16
modernc.org/sqlite modernc.org/sqlite
modernc.org/sqlite/lib modernc.org/sqlite/lib