Add options to import and export user data
This commit is contained in:
Omar Roth
parent
24c79a3554
commit
b535a9d413
@ -11,13 +11,16 @@ targets:
|
|||||||
dependencies:
|
dependencies:
|
||||||
kemal:
|
kemal:
|
||||||
github: kemalcr/kemal
|
github: kemalcr/kemal
|
||||||
branch: master
|
branch: rework-param-parser
|
||||||
pg:
|
pg:
|
||||||
github: will/crystal-pg
|
github: will/crystal-pg
|
||||||
branch: master
|
branch: master
|
||||||
detect_language:
|
detect_language:
|
||||||
github: detectlanguage/detectlanguage-crystal
|
github: detectlanguage/detectlanguage-crystal
|
||||||
branch: master
|
branch: master
|
||||||
|
sqlite3:
|
||||||
|
github: crystal-lang/crystal-sqlite3
|
||||||
|
branch: master
|
||||||
|
|
||||||
crystal: 0.25.1
|
crystal: 0.25.1
|
||||||
|
|
||||||
|
|||||||
181
src/invidious.cr
181
src/invidious.cr
@ -22,6 +22,7 @@ require "option_parser"
|
|||||||
require "pg"
|
require "pg"
|
||||||
require "xml"
|
require "xml"
|
||||||
require "yaml"
|
require "yaml"
|
||||||
|
require "zip"
|
||||||
require "./invidious/*"
|
require "./invidious/*"
|
||||||
|
|
||||||
CONFIG = Config.from_yaml(File.read("config/config.yml"))
|
CONFIG = Config.from_yaml(File.read("config/config.yml"))
|
||||||
@ -2182,15 +2183,195 @@ get "/subscription_manager" do |env|
|
|||||||
end
|
end
|
||||||
subscriptions = user.subscriptions
|
subscriptions = user.subscriptions
|
||||||
|
|
||||||
|
action_takeout = env.params.query["action_takeout"]?.try &.to_i?
|
||||||
|
action_takeout ||= 0
|
||||||
|
action_takeout = action_takeout == 1
|
||||||
|
|
||||||
|
format = env.params.query["format"]?
|
||||||
|
format ||= "rss"
|
||||||
|
|
||||||
client = make_client(YT_URL)
|
client = make_client(YT_URL)
|
||||||
subscriptions = subscriptions.map do |ucid|
|
subscriptions = subscriptions.map do |ucid|
|
||||||
get_channel(ucid, client, PG_DB, false)
|
get_channel(ucid, client, PG_DB, false)
|
||||||
end
|
end
|
||||||
subscriptions.sort_by! { |channel| channel.author.downcase }
|
subscriptions.sort_by! { |channel| channel.author.downcase }
|
||||||
|
|
||||||
|
if action_takeout
|
||||||
|
if Kemal.config.ssl || CONFIG.https_only
|
||||||
|
scheme = "https://"
|
||||||
|
else
|
||||||
|
scheme = "http://"
|
||||||
|
end
|
||||||
|
host = env.request.headers["Host"]
|
||||||
|
|
||||||
|
url = "#{scheme}#{host}"
|
||||||
|
|
||||||
|
if format == "json"
|
||||||
|
env.response.content_type = "application/json"
|
||||||
|
env.response.headers["content-disposition"] = "attachment"
|
||||||
|
next {
|
||||||
|
"subscriptions" => user.subscriptions,
|
||||||
|
"watch_history" => user.watched,
|
||||||
|
"preferences" => user.preferences,
|
||||||
|
}.to_json
|
||||||
|
else
|
||||||
|
env.response.content_type = "application/xml"
|
||||||
|
env.response.headers["content-disposition"] = "attachment"
|
||||||
|
export = XML.build do |xml|
|
||||||
|
xml.element("opml", version: "1.1") do
|
||||||
|
xml.element("body") do
|
||||||
|
if format == "newpipe"
|
||||||
|
title = "YouTube Subscriptions"
|
||||||
|
else
|
||||||
|
title = "Invidious Subscriptions"
|
||||||
|
end
|
||||||
|
|
||||||
|
xml.element("outline", text: title, title: title) do
|
||||||
|
subscriptions.each do |channel|
|
||||||
|
if format == "newpipe"
|
||||||
|
xmlUrl = "https://www.youtube.com/feeds/videos.xml?channel_id=#{channel.id}"
|
||||||
|
else
|
||||||
|
xmlUrl = "#{url}/feed/channel/#{channel.id}"
|
||||||
|
end
|
||||||
|
|
||||||
|
xml.element("outline", text: channel.author, title: channel.author,
|
||||||
|
"type": "rss", xmlUrl: xmlUrl)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
next export.gsub(%(<?xml version="1.0"?>\n), "")
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
templated "subscription_manager"
|
templated "subscription_manager"
|
||||||
end
|
end
|
||||||
|
|
||||||
|
get "/data_control" do |env|
|
||||||
|
user = env.get? "user"
|
||||||
|
referer = env.request.headers["referer"]?
|
||||||
|
referer ||= "/"
|
||||||
|
|
||||||
|
if user
|
||||||
|
user = user.as(User)
|
||||||
|
|
||||||
|
templated "data_control"
|
||||||
|
else
|
||||||
|
env.redirect referer
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
post "/data_control" do |env|
|
||||||
|
user = env.get? "user"
|
||||||
|
referer = env.request.headers["referer"]?
|
||||||
|
referer ||= "/"
|
||||||
|
|
||||||
|
if user
|
||||||
|
user = user.as(User)
|
||||||
|
|
||||||
|
HTTP::FormData.parse(env.request) do |part|
|
||||||
|
body = part.body.gets_to_end
|
||||||
|
if body.empty?
|
||||||
|
next
|
||||||
|
end
|
||||||
|
|
||||||
|
case part.name
|
||||||
|
when "import_invidious"
|
||||||
|
body = JSON.parse(body)
|
||||||
|
body["subscriptions"].as_a.each do |ucid|
|
||||||
|
ucid = ucid.as_s
|
||||||
|
if !user.subscriptions.includes? ucid
|
||||||
|
PG_DB.exec("UPDATE users SET subscriptions = array_append(subscriptions,$1) WHERE id = $2", ucid, user.id)
|
||||||
|
|
||||||
|
begin
|
||||||
|
client = make_client(YT_URL)
|
||||||
|
get_channel(ucid, client, PG_DB, false, false)
|
||||||
|
rescue ex
|
||||||
|
next
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
body["watch_history"].as_a.each do |id|
|
||||||
|
id = id.as_s
|
||||||
|
if !user.watched.includes? id
|
||||||
|
PG_DB.exec("UPDATE users SET watched = array_append(watched,$1) WHERE id = $2", id, user.id)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
PG_DB.exec("UPDATE users SET preferences = $1 WHERE id = $2", body["preferences"].to_json, user.id)
|
||||||
|
when "import_youtube"
|
||||||
|
subscriptions = XML.parse(body)
|
||||||
|
subscriptions.xpath_nodes(%q(//outline[@type="rss"])).each do |channel|
|
||||||
|
ucid = channel["xmlUrl"].match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0]
|
||||||
|
|
||||||
|
if !user.subscriptions.includes? ucid
|
||||||
|
PG_DB.exec("UPDATE users SET subscriptions = array_append(subscriptions,$1) WHERE id = $2", ucid, user.id)
|
||||||
|
|
||||||
|
begin
|
||||||
|
client = make_client(YT_URL)
|
||||||
|
get_channel(ucid, client, PG_DB, false, false)
|
||||||
|
rescue ex
|
||||||
|
next
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
when "import_newpipe_subscriptions"
|
||||||
|
body = JSON.parse(body)
|
||||||
|
body["subscriptions"].as_a.each do |channel|
|
||||||
|
ucid = channel["url"].as_s.match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0]
|
||||||
|
|
||||||
|
if !user.subscriptions.includes? ucid
|
||||||
|
PG_DB.exec("UPDATE users SET subscriptions = array_append(subscriptions,$1) WHERE id = $2", ucid, user.id)
|
||||||
|
|
||||||
|
begin
|
||||||
|
client = make_client(YT_URL)
|
||||||
|
get_channel(ucid, client, PG_DB, false, false)
|
||||||
|
rescue ex
|
||||||
|
next
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
when "import_newpipe"
|
||||||
|
Zip::Reader.open(body) do |file|
|
||||||
|
file.each_entry do |entry|
|
||||||
|
if entry.filename == "newpipe.db"
|
||||||
|
# We do this because the SQLite driver cannot parse a database from an IO
|
||||||
|
# Currently: channel URLs can **only** be subscriptions, and
|
||||||
|
# video URLs can **only** be watch history, so this works okay for now.
|
||||||
|
|
||||||
|
db = entry.io.gets_to_end
|
||||||
|
db.scan(/youtube\.com\/watch\?v\=(?<id>[a-zA-Z0-9_-]{11})/) do |md|
|
||||||
|
if !user.watched.includes? md["id"]
|
||||||
|
PG_DB.exec("UPDATE users SET watched = array_append(watched,$1) WHERE id = $2", md["id"], user.id)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
db.scan(/youtube\.com\/channel\/(?<ucid>[a-zA-Z0-9_-]{22})/) do |md|
|
||||||
|
ucid = md["ucid"]
|
||||||
|
if !user.subscriptions.includes? ucid
|
||||||
|
PG_DB.exec("UPDATE users SET subscriptions = array_append(subscriptions,$1) WHERE id = $2", ucid, user.id)
|
||||||
|
|
||||||
|
begin
|
||||||
|
client = make_client(YT_URL)
|
||||||
|
get_channel(ucid, client, PG_DB, false, false)
|
||||||
|
rescue ex
|
||||||
|
next
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
env.redirect referer
|
||||||
|
end
|
||||||
|
|
||||||
get "/subscription_ajax" do |env|
|
get "/subscription_ajax" do |env|
|
||||||
user = env.get? "user"
|
user = env.get? "user"
|
||||||
referer = env.request.headers["referer"]?
|
referer = env.request.headers["referer"]?
|
||||||
|
|||||||
50
src/invidious/views/data_control.ecr
Normal file
50
src/invidious/views/data_control.ecr
Normal file
@ -0,0 +1,50 @@
|
|||||||
|
<% content_for "header" do %>
|
||||||
|
<title>Import and Export Data - Invidious</title>
|
||||||
|
<% end %>
|
||||||
|
|
||||||
|
<div class="h-box">
|
||||||
|
<form class="pure-form pure-form-aligned" enctype="multipart/form-data" action="/data_control" method="post">
|
||||||
|
<fieldset>
|
||||||
|
<legend>Import</legend>
|
||||||
|
|
||||||
|
<div class="pure-control-group">
|
||||||
|
<label for="import_youtube">Import Invidious data</label>
|
||||||
|
<input type="file" id="import_invidious" name="import_invidious">
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="pure-control-group">
|
||||||
|
<label for="import_youtube">Import <a target="_blank" style="color: #0366d6"
|
||||||
|
href="https://support.google.com/youtube/answer/6224202?hl=en-GB">YouTube subscriptions</a></label>
|
||||||
|
<input type="file" id="import_youtube" name="import_youtube">
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="pure-control-group">
|
||||||
|
<label for="import_newpipe_subscriptions">Import NewPipe subscriptions (.json)</label>
|
||||||
|
<input type="file" id="import_newpipe_subscriptions" name="import_newpipe_subscriptions">
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="pure-control-group">
|
||||||
|
<label for="import_newpipe">Import NewPipe data (.zip)</label>
|
||||||
|
<input type="file" id="import_newpipe" name="import_newpipe">
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="pure-controls">
|
||||||
|
<button type="submit" class="pure-button pure-button-primary">Import</button>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<legend>Export</legend>
|
||||||
|
|
||||||
|
<div class="pure-control-group">
|
||||||
|
<a href="/subscription_manager?action_takeout=1">Export subscriptions as OPML</a>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="pure-control-group">
|
||||||
|
<a href="/subscription_manager?action_takeout=1&format=newpipe">Export subscriptions as OPML (NewPipe)</a>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="pure-control-group">
|
||||||
|
<a href="/subscription_manager?action_takeout=1&format=json">Export data as JSON</a>
|
||||||
|
</div>
|
||||||
|
</fieldset>
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
@ -101,7 +101,13 @@ function update_value(element) {
|
|||||||
<div class="pure-control-group">
|
<div class="pure-control-group">
|
||||||
<label>
|
<label>
|
||||||
<a href="/clear_watch_history">Clear watch history</a>
|
<a href="/clear_watch_history">Clear watch history</a>
|
||||||
</labe>
|
</label>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="pure-control-group">
|
||||||
|
<label>
|
||||||
|
<a href="/data_control">Import/Export data</a>
|
||||||
|
</label>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="pure-controls">
|
<div class="pure-controls">
|
||||||
|
|||||||
@ -2,7 +2,16 @@
|
|||||||
<title>Subscription manager - Invidious</title>
|
<title>Subscription manager - Invidious</title>
|
||||||
<% end %>
|
<% end %>
|
||||||
|
|
||||||
<h1><%= subscriptions.size %> subscriptions</h1>
|
<div class="pure-g h-box">
|
||||||
|
<div class="pure-u-2-3">
|
||||||
|
<h3><%= subscriptions.size %> subscriptions</h3>
|
||||||
|
</div>
|
||||||
|
<div class="pure-u-1-3" style="text-align:right;">
|
||||||
|
<h3>
|
||||||
|
<a href="/data_control">Import/Export</a>
|
||||||
|
</h3>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
<% subscriptions.each do |channel| %>
|
<% subscriptions.each do |channel| %>
|
||||||
<h3 class="h-box">
|
<h3 class="h-box">
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user