lug/invidious
7
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Clean up data import/export

Browse Source
This commit is contained in:
Omar Roth 2018-11-09 17:25:24 -06:00
parent 6066615553
commit b77c73df0d
2 changed files with 82 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
shard.yml
View File

@ -13,7 +13,6 @@ dependencies:
github: detectlanguage/detectlanguage-crystal github: detectlanguage/detectlanguage-crystal
kemal: kemal:
github: kemalcr/kemal github: kemalcr/kemal
commit: b389022
pg: pg:
github: will/crystal-pg github: will/crystal-pg

149
src/invidious.cr
View File

@ -1230,83 +1230,85 @@ post "/data_control" do |env|
case part.name case part.name
when "import_invidious" when "import_invidious"
body = JSON.parse(body) body = JSON.parse(body)
body["subscriptions"]?.try &.as_a.each do |ucid|
ucid = ucid.as_s
if !user.subscriptions.includes? ucid if body["subscriptions"]?
user.subscriptions += body["subscriptions"].as_a.map { |a| a.as_s }
user.subscriptions.uniq!
user.subscriptions.each do |ucid|
begin begin
client = make_client(YT_URL) client = make_client(YT_URL)
get_channel(ucid, client, PG_DB, false, false) get_channel(ucid, client, PG_DB, false, false)
PG_DB.exec("UPDATE users SET subscriptions = array_append(subscriptions,$1) WHERE email = $2", ucid, user.email)
user.subscriptions << ucid
rescue ex rescue ex
next next
end end
end end
PG_DB.exec("UPDATE users SET subscriptions = $1 WHERE email = $2", user.subscriptions, user.email)
end end
if body["watch_history"]? if body["watch_history"]?
watched = user.watched + body["watch_history"].as_a.map { |a| a.as_s } user.watched += body["watch_history"].as_a.map { |a| a.as_s }
watched.uniq! user.watched.uniq!
PG_DB.exec("UPDATE users SET watched = $1 WHERE email = $2", watched, user.email) PG_DB.exec("UPDATE users SET watched = $1 WHERE email = $2", user.watched, user.email)
end end
if body["preferences"]? if body["preferences"]?
PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", body["preferences"].to_json, user.email) user.preferences = Preferences.from_json(body["preferences"].to_json)
PG_DB.exec("UPDATE users SET preferences = $1 WHERE email = $2", user.preferences.to_json, user.email)
end end
when "import_youtube" when "import_youtube"
subscriptions = XML.parse(body) subscriptions = XML.parse(body)
subscriptions.xpath_nodes(%q(//outline[@type="rss"])).each do |channel| user.subscriptions += subscriptions.xpath_nodes(%q(//outline[@type="rss"])).map do |channel|
ucid = channel["xmlUrl"].match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0] channel["xmlUrl"].match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0]
end
user.subscriptions.uniq!
if !user.subscriptions.includes? ucid user.subscriptions.each do |ucid|
begin begin
client = make_client(YT_URL) client = make_client(YT_URL)
get_channel(ucid, client, PG_DB, false, false) get_channel(ucid, client, PG_DB, false, false)
rescue ex
PG_DB.exec("UPDATE users SET subscriptions = array_append(subscriptions,$1) WHERE email = $2", ucid, user.email) next
user.subscriptions << ucid
rescue ex
next
end
end end
end end
PG_DB.exec("UPDATE users SET subscriptions = $1 WHERE email = $2", user.subscriptions, user.email)
when "import_freetube" when "import_freetube"
body.scan(/"channelId":"(?<channel_id>[a-zA-Z0-9_-]{24})"/).each do |md| user.subscriptions += body.scan(/"channelId":"(?<channel_id>[a-zA-Z0-9_-]{24})"/).map do |md|
ucid = md["channel_id"] md["channel_id"]
end
user.subscriptions.uniq!
if !user.subscriptions.includes? ucid user.subscriptions.each do |ucid|
begin begin
client = make_client(YT_URL) client = make_client(YT_URL)
get_channel(ucid, client, PG_DB, false, false) get_channel(ucid, client, PG_DB, false, false)
rescue ex
PG_DB.exec("UPDATE users SET subscriptions = array_append(subscriptions,$1) WHERE email = $2", ucid, user.email) next
user.subscriptions << ucid
rescue ex
next
end
end end
end end
PG_DB.exec("UPDATE users SET subscriptions = $1 WHERE email = $2", user.subscriptions, user.email)
when "import_newpipe_subscriptions" when "import_newpipe_subscriptions"
body = JSON.parse(body) body = JSON.parse(body)
body["subscriptions"].as_a.each do |channel| user.subscriptions += body["subscriptions"].as_a.map do |channel|
ucid = channel["url"].as_s.match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0] channel["url"].as_s.match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0]
end
user.subscriptions.uniq!
if !user.subscriptions.includes? ucid user.subscriptions.each do |ucid|
begin begin
client = make_client(YT_URL) client = make_client(YT_URL)
get_channel(ucid, client, PG_DB, false, false) get_channel(ucid, client, PG_DB, false, false)
rescue ex
PG_DB.exec("UPDATE users SET subscriptions = array_append(subscriptions,$1) WHERE email = $2", ucid, user.email) next
user.subscriptions << ucid
rescue ex
next
end
end end
end end
PG_DB.exec("UPDATE users SET subscriptions = $1 WHERE email = $2", user.subscriptions, user.email)
when "import_newpipe" when "import_newpipe"
Zip::Reader.open(body) do |file| Zip::Reader.open(IO::Memory.new(body)) do |file|
file.each_entry do |entry| file.each_entry do |entry|
if entry.filename == "newpipe.db" if entry.filename == "newpipe.db"
# We do this because the SQLite driver cannot parse a database from an IO # We do this because the SQLite driver cannot parse a database from an IO
@ -1314,30 +1316,29 @@ post "/data_control" do |env|
# video URLs can **only** be watch history, so this works okay for now. # video URLs can **only** be watch history, so this works okay for now.
db = entry.io.gets_to_end db = entry.io.gets_to_end
db.scan(/youtube\.com\/watch\?v\=(?<id>[a-zA-Z0-9_-]{11})/) do |md|
id = md["id"]
if !user.watched.includes? id user.watched += db.scan(/youtube\.com\/watch\?v\=(?<id>[a-zA-Z0-9_-]{11})/).map do |md|
PG_DB.exec("UPDATE users SET watched = array_append(watched,$1) WHERE email = $2", id, user.email) md["id"]
user.watched << id end
user.watched.uniq!
PG_DB.exec("UPDATE users SET watched = $1 WHERE email = $2", user.watched, user.email)
user.subscriptions += db.scan(/youtube\.com\/channel\/(?<ucid>[a-zA-Z0-9_-]{22})/).map do |md|
md["ucid"]
end
user.subscriptions.uniq!
user.subscriptions.each do |ucid|
begin
client = make_client(YT_URL)
get_channel(ucid, client, PG_DB, false, false)
rescue ex
next
end end
end end
db.scan(/youtube\.com\/channel\/(?<ucid>[a-zA-Z0-9_-]{22})/) do |md| PG_DB.exec("UPDATE users SET subscriptions = $1 WHERE email = $2", user.subscriptions, user.email)
ucid = md["ucid"]
if !user.subscriptions.includes? ucid
begin
client = make_client(YT_URL)
get_channel(ucid, client, PG_DB, false, false)
PG_DB.exec("UPDATE users SET subscriptions = array_append(subscriptions,$1) WHERE email = $2", ucid, user.email)
user.subscriptions << ucid
rescue ex
next
end
end
end
end end
end end
end end
@ -1629,6 +1630,20 @@ get "/feed/subscriptions" do |env|
end end
end end
get "/feed/history" do |env|
user = env.get? "user"
referer = get_referer(env)
if user
user = user.as(User)
watched = user.watched.reverse
templated "history"
else
env.redirect referer
end
end
get "/feed/channel/:ucid" do |env| get "/feed/channel/:ucid" do |env|
env.response.content_type = "text/xml" env.response.content_type = "text/xml"
ucid = env.params.url["ucid"] ucid = env.params.url["ucid"]