lug/invidious
7
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #2827 from SamantazFox/more-code-cleanup

Browse Source 
More code cleanup
This commit is contained in:
Samantaz Fox 2022-02-02 00:36:19 +01:00 committed by GitHub
commit fc5f84a0cd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
30 changed files with 193 additions and 305 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
scripts/git/pre-commit
View File

@ -15,9 +15,3 @@ if [ ! -z "$changed_cr_files" ]; then
git add $changed_cr_files
fi
# Locale equalizer
if [ ! -z $(git diff --name-only --cached -- locales/) ]; then
crystal run scripts/propagate-new-locale-keys.cr
git add locales > /dev/null
fi

95
scripts/propagate-new-locale-keys.cr
View File

@ -1,95 +0,0 @@
require "json"
require "../src/invidious/helpers/i18n.cr"
def locale_to_array(locale_name)
arrayifed_locale_data = [] of Tuple(String, JSON::Any | String)
keys_only_array = [] of String
LOCALES[locale_name].each do |k, v|
if v.as_h?
arrayifed_locale_data << {k, JSON.parse(v.as_h.to_json)}
elsif v.as_s?
arrayifed_locale_data << {k, v.as_s}
end
keys_only_array << k
end
return arrayifed_locale_data, keys_only_array
end
# Invidious currently has some unloaded localization files. We shouldn't need to propagate new keys onto those.
# We'll also remove the reference locale (english) from the list to process.
loaded_locales = LOCALES.keys.select! { |key| key != "en-US" }
english_locale, english_locale_keys = locale_to_array("en-US")
# In order to automatically propagate locale keys we're going to be needing two arrays.
# One is an array containing each locale data encoded as tuples. The other would contain
# sets of only the keys of each locale files.
#
# The second array is to make sure that an key from the english reference file is present
# in whatever the current locale we're scanning is.
locale_list = [] of Array(Tuple(String, JSON::Any | String))
locale_list_with_only_keys = [] of Array(String)
# Populates the created arrays from above
loaded_locales.each do |name|
arrayifed_locale_data, keys_only_locale = locale_to_array(name)
locale_list << arrayifed_locale_data
locale_list_with_only_keys << keys_only_locale
end
# Propagate additions
locale_list_with_only_keys.dup.each_with_index do |keys_of_locale_in_processing, index_of_locale_in_processing|
insert_at = {} of Int32 => Tuple(String, JSON::Any | String)
LOCALES["en-US"].each_with_index do |ref_locale_data, ref_locale_key_index|
ref_locale_key, ref_locale_value = ref_locale_data
# Found an new key that isn't present in the current locale..
if !keys_of_locale_in_processing.includes? ref_locale_key
# In terms of structure there's currently only two types; one for plural and the other for singular translations.
if ref_locale_value.as_h?
insert_at[ref_locale_key_index] = {ref_locale_key, JSON.parse({"([^.,0-9]|^)1([^.,0-9]|$)" => "", "" => ""}.to_json)}
else
insert_at[ref_locale_key_index] = {ref_locale_key, ""}
end
end
end
insert_at.each do |location_to_insert, data|
locale_list_with_only_keys[index_of_locale_in_processing].insert(location_to_insert, data[0])
locale_list[index_of_locale_in_processing].insert(location_to_insert, data)
end
end
# Propagate removals
locale_list_with_only_keys.dup.each_with_index do |keys_of_locale_in_processing, index_of_locale_in_processing|
remove_at = [] of Int32
keys_of_locale_in_processing.each_with_index do |current_key, current_key_index|
if !english_locale_keys.includes? current_key
remove_at << current_key_index
end
end
remove_at.each do |index_to_remove_at|
locale_list_with_only_keys[index_of_locale_in_processing].delete_at(index_to_remove_at)
locale_list[index_of_locale_in_processing].delete_at(index_to_remove_at)
end
end
# Now we convert back to our original format.
final_locale_list = [] of String
locale_list.each do |locale|
intermediate_hash = {} of String => (JSON::Any | String)
locale.each { |k, v| intermediate_hash[k] = v }
final_locale_list << intermediate_hash.to_pretty_json(indent = " ")
end
locale_map = Hash.zip(loaded_locales, final_locale_list)
# Export
locale_map.each do |locale_name, locale_contents|
File.write("locales/#{locale_name}.json", "#{locale_contents}\n")
end

5
src/invidious/channels/about.cr
View File

@ -96,7 +96,7 @@ def get_about_info(ucid, locale) : AboutChannel
total_views = channel_about_meta["viewCountText"]?.try &.["simpleText"]?.try &.as_s.gsub(/\D/, "").to_i64? || 0_i64
# The joined text is split to several sub strings. The reduce joins those strings before parsing the date.
joined = channel_about_meta["joinedDateText"]?.try &.["runs"]?.try &.as_a.reduce("") { |acc, node| acc + node["text"].as_s }
joined = channel_about_meta["joinedDateText"]?.try &.["runs"]?.try &.as_a.reduce("") { |acc, nd| acc + nd["text"].as_s }
.try { |text| Time.parse(text, "Joined %b %-d, %Y", Time::Location.local) } || Time.unix(0)
# Normal Auto-generated channels
@ -136,7 +136,8 @@ def fetch_related_channels(about_channel : AboutChannel) : Array(AboutRelatedCha
channels = YoutubeAPI.browse(browse_id: about_channel.ucid, params: "EghjaGFubmVscw%3D%3D")
tabs = channels.dig?("contents", "twoColumnBrowseResultsRenderer", "tabs").try(&.as_a?) || [] of JSON::Any
tab = tabs.find { |tab| tab.dig?("tabRenderer", "title").try(&.as_s?) == "Channels" }
tab = tabs.find(&.dig?("tabRenderer", "title").try(&.as_s?).try(&.== "Channels"))
return [] of AboutRelatedChannel if tab.nil?
items = tab.dig?("tabRenderer", "content", "sectionListRenderer", "contents", 0, "itemSectionRenderer", "contents", 0, "gridRenderer", "items").try(&.as_a?) || [] of JSON::Any

12
src/invidious/channels/channels.cr
View File

@ -44,15 +44,11 @@ struct ChannelVideo
end
end
def to_json(locale, json : JSON::Builder | Nil = nil)
if json
to_json(locale, json)
else
def to_json(locale, _json : Nil = nil)
JSON.build do |json|
to_json(locale, json)
end
end
end
def to_xml(locale, query_params, xml : XML::Builder)
query_params["v"] = self.id
@ -88,15 +84,11 @@ struct ChannelVideo
end
end
def to_xml(locale, xml : XML::Builder | Nil = nil)
if xml
to_xml(locale, xml)
else
def to_xml(locale, _xml : Nil = nil)
XML.build do |xml|
to_xml(locale, xml)
end
end
end
def to_tuple
{% begin %}

4
src/invidious/comments.cr
View File

@ -93,10 +93,6 @@ def fetch_youtube_comments(id, cursor, format, locale, thin_mode, region, sort_b
end
contents = body["contents"]?
header = body["header"]?
if body["continuations"]?
# Removable? Doesn't seem like this is used.
more_replies_continuation = body["continuations"][0]["nextContinuationData"]["continuation"].as_s
end
else
raise InfoException.new("Could not fetch comments")
end

78
src/invidious/helpers/errors.cr
View File

@ -6,8 +6,12 @@
class InfoException < Exception
end
# -------------------
# Issue template
# -------------------
macro error_template(*args)
error_template_helper(env, locale, {{*args}})
error_template_helper(env, {{*args}})
end
def github_details(summary : String, content : String)
@ -22,11 +26,13 @@ def github_details(summary : String, content : String)
return HTML.escape(details)
end
def error_template_helper(env : HTTP::Server::Context, locale : String?, status_code : Int32, exception : Exception)
def error_template_helper(env : HTTP::Server::Context, status_code : Int32, exception : Exception)
if exception.is_a?(InfoException)
return error_template_helper(env, locale, status_code, exception.message || "")
return error_template_helper(env, status_code, exception.message || "")
end
locale = env.get("preferences").as(Preferences).locale
env.response.content_type = "text/html"
env.response.status_code = status_code
@ -77,71 +83,101 @@ def error_template_helper(env : HTTP::Server::Context, locale : String?, status_
return templated "error"
end
def error_template_helper(env : HTTP::Server::Context, locale : String?, status_code : Int32, message : String)
def error_template_helper(env : HTTP::Server::Context, status_code : Int32, message : String)
env.response.content_type = "text/html"
env.response.status_code = status_code
locale = env.get("preferences").as(Preferences).locale
error_message = translate(locale, message)
next_steps = error_redirect_helper(env, locale)
next_steps = error_redirect_helper(env)
return templated "error"
end
# -------------------
# Atom feeds
# -------------------
macro error_atom(*args)
error_atom_helper(env, locale, {{*args}})
error_atom_helper(env, {{*args}})
end
def error_atom_helper(env : HTTP::Server::Context, locale : String?, status_code : Int32, exception : Exception)
def error_atom_helper(env : HTTP::Server::Context, status_code : Int32, exception : Exception)
if exception.is_a?(InfoException)
return error_atom_helper(env, locale, status_code, exception.message || "")
return error_atom_helper(env, status_code, exception.message || "")
end
env.response.content_type = "application/atom+xml"
env.response.status_code = status_code
return "<error>#{exception.inspect_with_backtrace}</error>"
end
def error_atom_helper(env : HTTP::Server::Context, locale : String?, status_code : Int32, message : String)
def error_atom_helper(env : HTTP::Server::Context, status_code : Int32, message : String)
env.response.content_type = "application/atom+xml"
env.response.status_code = status_code
return "<error>#{message}</error>"
end
# -------------------
# JSON
# -------------------
macro error_json(*args)
error_json_helper(env, locale, {{*args}})
error_json_helper(env, {{*args}})
end
def error_json_helper(env : HTTP::Server::Context, locale : String?, status_code : Int32, exception : Exception, additional_fields : Hash(String, Object) | Nil)
def error_json_helper(
env : HTTP::Server::Context,
status_code : Int32,
exception : Exception,
additional_fields : Hash(String, Object) | Nil = nil
)
if exception.is_a?(InfoException)
return error_json_helper(env, locale, status_code, exception.message || "", additional_fields)
return error_json_helper(env, status_code, exception.message || "", additional_fields)
end
env.response.content_type = "application/json"
env.response.status_code = status_code
error_message = {"error" => exception.message, "errorBacktrace" => exception.inspect_with_backtrace}
if additional_fields
error_message = error_message.merge(additional_fields)
end
return error_message.to_json
end
def error_json_helper(env : HTTP::Server::Context, locale : String?, status_code : Int32, exception : Exception)
return error_json_helper(env, locale, status_code, exception, nil)
end
def error_json_helper(env : HTTP::Server::Context, locale : String?, status_code : Int32, message : String, additional_fields : Hash(String, Object) | Nil)
def error_json_helper(
env : HTTP::Server::Context,
status_code : Int32,
message : String,
additional_fields : Hash(String, Object) | Nil = nil
)
env.response.content_type = "application/json"
env.response.status_code = status_code
error_message = {"error" => message}
if additional_fields
error_message = error_message.merge(additional_fields)
end
return error_message.to_json
end
def error_json_helper(env : HTTP::Server::Context, locale : String?, status_code : Int32, message : String)
error_json_helper(env, locale, status_code, message, nil)
end
# -------------------
# Redirect
# -------------------
def error_redirect_helper(env : HTTP::Server::Context, locale : String?)
def error_redirect_helper(env : HTTP::Server::Context)
request_path = env.request.path
locale = env.get("preferences").as(Preferences).locale
if request_path.starts_with?("/search") || request_path.starts_with?("/watch") ||
request_path.starts_with?("/channel") || request_path.starts_with?("/playlist?list=PL")
next_steps_text = translate(locale, "next_steps_error_message")

4
src/invidious/helpers/i18n.cr
View File

@ -94,8 +94,8 @@ def translate(locale : String?, key : String, text : String | Nil = nil) : Strin
translation = ""
match_length = 0
raw_data.as_h.each do |key, value|
if md = text.try &.match(/#{key}/)
raw_data.as_h.each do |hash_key, value|
if md = text.try &.match(/#{hash_key}/)
if md[0].size >= match_length
translation = value.as_s
match_length = md[0].size

4
src/invidious/helpers/json_filter.cr
View File

@ -98,9 +98,9 @@ module JSONFilter
end
end
group_name.split('/').each do |group_name|
group_name.split('/').each do |name|
nest_stack.push({
group_name: group_name,
group_name: name,
closing_bracket_index: closing_bracket_index,
})
end

5
src/invidious/helpers/static_file_handler.cr
View File

@ -175,9 +175,8 @@ module Kemal
if @cached_files.sum(&.[1][:data].bytesize) + (size = File.size(file_path)) < CACHE_LIMIT
data = Bytes.new(size)
File.open(file_path) do |file|
file.read(data)
end
File.open(file_path, &.read(data))
filestat = File.info(file_path)
@cached_files[file_path] = {data: data, filestat: filestat}

3
src/invidious/helpers/tokens.cr
View File

@ -42,6 +42,9 @@ end
def sign_token(key, hash)
string_to_sign = [] of String
# TODO: figure out which "key" variable is used
# Ameba reports a warning for "Lint/ShadowingOuterLocalVar" on this
# variable, but its preferrable to not touch that (works fine atm).
hash.each do |key, value|
next if key == "signature"

4
src/invidious/helpers/utils.cr
View File

@ -292,8 +292,8 @@ def parse_range(range)
end
ranges = range.lchop("bytes=").split(',')
ranges.each do |range|
start_range, end_range = range.split('-')
ranges.each do |r|
start_range, end_range = r.split('-')
start_range = start_range.to_i64? || 0_i64
end_range = end_range.to_i64?

59
src/invidious/playlists.cr
View File

@ -90,7 +90,7 @@ struct Playlist
property updated : Time
property thumbnail : String?
def to_json(offset, locale, json : JSON::Builder, video_id : String? = nil)
def to_json(offset, json : JSON::Builder, video_id : String? = nil)
json.object do
json.field "type", "playlist"
json.field "title", self.title
@ -125,7 +125,7 @@ struct Playlist
json.field "videos" do
json.array do
videos = get_playlist_videos(self, offset: offset, locale: locale, video_id: video_id)
videos = get_playlist_videos(self, offset: offset, video_id: video_id)
videos.each do |video|
video.to_json(json)
end
@ -134,13 +134,9 @@ struct Playlist
end
end
def to_json(offset, locale, json : JSON::Builder? = nil, video_id : String? = nil)
if json
to_json(offset, locale, json, video_id: video_id)
else
def to_json(offset, _json : Nil = nil, video_id : String? = nil)
JSON.build do |json|
to_json(offset, locale, json, video_id: video_id)
end
to_json(offset, json, video_id: video_id)
end
end
@ -179,7 +175,7 @@ struct InvidiousPlaylist
end
end
def to_json(offset, locale, json : JSON::Builder, video_id : String? = nil)
def to_json(offset, json : JSON::Builder, video_id : String? = nil)
json.object do
json.field "type", "invidiousPlaylist"
json.field "title", self.title
@ -205,22 +201,18 @@ struct InvidiousPlaylist
offset = self.index.index(index) || 0
end
videos = get_playlist_videos(self, offset: offset, locale: locale, video_id: video_id)
videos.each_with_index do |video, index|
video.to_json(json, offset + index)
videos = get_playlist_videos(self, offset: offset, video_id: video_id)
videos.each_with_index do |video, idx|
video.to_json(json, offset + idx)
end
end
end
end
end
def to_json(offset, locale, json : JSON::Builder? = nil, video_id : String? = nil)
if json
to_json(offset, locale, json, video_id: video_id)
else
def to_json(offset, _json : Nil = nil, video_id : String? = nil)
JSON.build do |json|
to_json(offset, locale, json, video_id: video_id)
end
to_json(offset, json, video_id: video_id)
end
end
@ -320,7 +312,7 @@ def produce_playlist_continuation(id, index)
return continuation
end
def get_playlist(plid, locale, refresh = true, force_refresh = false)
def get_playlist(plid : String)
if plid.starts_with? "IV"
if playlist = Invidious::Database::Playlists.select(id: plid)
return playlist
@ -328,21 +320,21 @@ def get_playlist(plid, locale, refresh = true, force_refresh = false)
raise InfoException.new("Playlist does not exist.")
end
else
return fetch_playlist(plid, locale)
return fetch_playlist(plid)
end
end
def fetch_playlist(plid, locale)
def fetch_playlist(plid : String)
if plid.starts_with? "UC"
plid = "UU#{plid.lchop("UC")}"
end
initial_data = YoutubeAPI.browse("VL" + plid, params: "")
playlist_sidebar_renderer = initial_data["sidebar"]?.try &.["playlistSidebarRenderer"]?.try &.["items"]?
playlist_sidebar_renderer = initial_data.dig?("sidebar", "playlistSidebarRenderer", "items")
raise InfoException.new("Could not extract playlistSidebarRenderer.") if !playlist_sidebar_renderer
playlist_info = playlist_sidebar_renderer[0]["playlistSidebarPrimaryInfoRenderer"]?
playlist_info = playlist_sidebar_renderer.dig?(0, "playlistSidebarPrimaryInfoRenderer")
raise InfoException.new("Could not extract playlist info") if !playlist_info
title = playlist_info.dig?("title", "runs", 0, "text").try &.as_s || ""
@ -355,12 +347,15 @@ def fetch_playlist(plid, locale)
description_html = desc_item.try &.["runs"]?.try &.as_a
.try { |run| content_to_comment_html(run).try &.to_s } || "<p></p>"
thumbnail = playlist_info["thumbnailRenderer"]?.try &.["playlistVideoThumbnailRenderer"]?
.try &.["thumbnail"]["thumbnails"][0]["url"]?.try &.as_s
thumbnail = playlist_info.dig?(
"thumbnailRenderer", "playlistVideoThumbnailRenderer",
"thumbnail", "thumbnails", 0, "url"
).try &.as_s
views = 0_i64
updated = Time.utc
video_count = 0
playlist_info["stats"]?.try &.as_a.each do |stat|
text = stat["runs"]?.try &.as_a.map(&.["text"].as_s).join("") || stat["simpleText"]?.try &.as_s
next if !text
@ -379,12 +374,15 @@ def fetch_playlist(plid, locale)
author_thumbnail = ""
ucid = ""
else
author_info = playlist_sidebar_renderer[1]["playlistSidebarSecondaryInfoRenderer"]?.try &.["videoOwner"]["videoOwnerRenderer"]?
author_info = playlist_sidebar_renderer[1].dig?(
"playlistSidebarSecondaryInfoRenderer", "videoOwner", "videoOwnerRenderer"
)
raise InfoException.new("Could not extract author info") if !author_info
author = author_info["title"]["runs"][0]["text"]?.try &.as_s || ""
author_thumbnail = author_info["thumbnail"]["thumbnails"][0]["url"]?.try &.as_s || ""
ucid = author_info["title"]["runs"][0]["navigationEndpoint"]["browseEndpoint"]["browseId"]?.try &.as_s || ""
author = author_info.dig?("title", "runs", 0, "text").try &.as_s || ""
author_thumbnail = author_info.dig?("thumbnail", "thumbnails", 0, "url").try &.as_s || ""
ucid = author_info.dig?("title", "runs", 0, "navigationEndpoint", "browseEndpoint", "browseId").try &.as_s || ""
end
return Playlist.new({
@ -402,7 +400,7 @@ def fetch_playlist(plid, locale)
})
end
def get_playlist_videos(playlist, offset, locale = nil, video_id = nil)
def get_playlist_videos(playlist : InvidiousPlaylist | Playlist, offset : Int32, video_id = nil)
# Show empy playlist if requested page is out of range
# (e.g, when a new playlist has been created, offset will be negative)
if offset >= playlist.video_count || offset < 0
@ -465,7 +463,6 @@ def extract_playlist_videos(initial_data : Hash(String, JSON::Any))
plid = i["navigationEndpoint"]["watchEndpoint"]["playlistId"].as_s
index = i["navigationEndpoint"]["watchEndpoint"]["index"].as_i64
thumbnail = i["thumbnail"]["thumbnails"][0]["url"].as_s
title = i["title"].try { |t| t["simpleText"]? || t["runs"]?.try &.[0]["text"]? }.try &.as_s || ""
author = i["shortBylineText"]?.try &.["runs"][0]["text"].as_s || ""
ucid = i["shortBylineText"]?.try &.["runs"][0]["navigationEndpoint"]["browseEndpoint"]["browseId"].as_s || ""

2
src/invidious/routes/api/manifest.cr
View File

@ -98,7 +98,7 @@ module Invidious::Routes::API::Manifest
height = fmt["height"].as_i
# Resolutions reported by YouTube player (may not accurately reflect source)
height = potential_heights.min_by { |i| (height - i).abs }
height = potential_heights.min_by { |x| (height - x).abs }
next if unique_res && heights.includes? height
heights << height

19
src/invidious/routes/api/v1/authenticated.cr
View File

@ -115,8 +115,6 @@ module Invidious::Routes::API::V1::Authenticated
end
def self.list_playlists(env)
locale = env.get("preferences").as(Preferences).locale
env.response.content_type = "application/json"
user = env.get("user").as(User)
@ -125,7 +123,7 @@ module Invidious::Routes::API::V1::Authenticated
JSON.build do |json|
json.array do
playlists.each do |playlist|
playlist.to_json(0, locale, json)
playlist.to_json(0, json)
end
end
end
@ -134,14 +132,13 @@ module Invidious::Routes::API::V1::Authenticated
def self.create_playlist(env)
env.response.content_type = "application/json"
user = env.get("user").as(User)
locale = env.get("preferences").as(Preferences).locale
title = env.params.json["title"]?.try &.as(String).delete("<>").byte_slice(0, 150)
if !title
return error_json(400, "Invalid title.")
end
privacy = env.params.json["privacy"]?.try { |privacy| PlaylistPrivacy.parse(privacy.as(String).downcase) }
privacy = env.params.json["privacy"]?.try { |p| PlaylistPrivacy.parse(p.as(String).downcase) }
if !privacy
return error_json(400, "Invalid privacy setting.")
end
@ -160,8 +157,6 @@ module Invidious::Routes::API::V1::Authenticated
end
def self.update_playlist_attribute(env)
locale = env.get("preferences").as(Preferences).locale
env.response.content_type = "application/json"
user = env.get("user").as(User)
@ -180,7 +175,7 @@ module Invidious::Routes::API::V1::Authenticated
end
title = env.params.json["title"].try &.as(String).delete("<>").byte_slice(0, 150) || playlist.title
privacy = env.params.json["privacy"]?.try { |privacy| PlaylistPrivacy.parse(privacy.as(String).downcase) } || playlist.privacy
privacy = env.params.json["privacy"]?.try { |p| PlaylistPrivacy.parse(p.as(String).downcase) } || playlist.privacy
description = env.params.json["description"]?.try &.as(String).delete("\r") || playlist.description
if title != playlist.title ||
@ -197,8 +192,6 @@ module Invidious::Routes::API::V1::Authenticated
end
def self.delete_playlist(env)
locale = env.get("preferences").as(Preferences).locale
env.response.content_type = "application/json"
user = env.get("user").as(User)
@ -219,8 +212,6 @@ module Invidious::Routes::API::V1::Authenticated
end
def self.insert_video_into_playlist(env)
locale = env.get("preferences").as(Preferences).locale
env.response.content_type = "application/json"
user = env.get("user").as(User)
@ -274,8 +265,6 @@ module Invidious::Routes::API::V1::Authenticated
end
def self.delete_video_in_playlist(env)
locale = env.get("preferences").as(Preferences).locale
env.response.content_type = "application/json"
user = env.get("user").as(User)
@ -389,8 +378,8 @@ module Invidious::Routes::API::V1::Authenticated
end
def self.unregister_token(env)
locale = env.get("preferences").as(Preferences).locale
env.response.content_type = "application/json"
user = env.get("user").as(User)
scopes = env.get("scopes").as(Array(String))

2
src/invidious/routes/api/v1/channels.cr
View File

@ -254,7 +254,7 @@ module Invidious::Routes::API::V1::Channels
page = env.params.query["page"]?.try &.to_i?
page ||= 1
count, search_results = channel_search(query, page, ucid)
search_results = channel_search(query, page, ucid)
JSON.build do |json|
json.array do
search_results.each do |item|

13
src/invidious/routes/api/v1/misc.cr
View File

@ -1,7 +1,6 @@
module Invidious::Routes::API::V1::Misc
# Stats API endpoint for Invidious
def self.stats(env)
locale = env.get("preferences").as(Preferences).locale
env.response.content_type = "application/json"
if !CONFIG.statistics_enabled
@ -14,9 +13,7 @@ module Invidious::Routes::API::V1::Misc
# APIv1 currently uses the same logic for both
# user playlists and Invidious playlists. This means that we can't
# reasonably split them yet. This should be addressed in APIv2
def self.get_playlist(env)
locale = env.get("preferences").as(Preferences).locale
def self.get_playlist(env : HTTP::Server::Context)
env.response.content_type = "application/json"
plid = env.params.url["plid"]
@ -34,7 +31,7 @@ module Invidious::Routes::API::V1::Misc
end
begin
playlist = get_playlist(plid, locale)
playlist = get_playlist(plid)
rescue ex : InfoException
return error_json(404, ex)
rescue ex
@ -49,7 +46,7 @@ module Invidious::Routes::API::V1::Misc
# includes into the playlist a maximum of 20 videos, before the offset
if offset > 0
lookback = offset < 50 ? offset : 50
response = playlist.to_json(offset - lookback, locale)
response = playlist.to_json(offset - lookback)
json_response = JSON.parse(response)
else
# Unless the continuation is really the offset 0, it becomes expensive.
@ -58,13 +55,13 @@ module Invidious::Routes::API::V1::Misc
# it shouldn't happen often though
lookback = 0
response = playlist.to_json(offset, locale, video_id: video_id)
response = playlist.to_json(offset, video_id: video_id)
json_response = JSON.parse(response)
if json_response["videos"].as_a[0]["index"] != offset
offset = json_response["videos"].as_a[0]["index"].as_i
lookback = offset < 50 ? offset : 50
response = playlist.to_json(offset - lookback, locale)
response = playlist.to_json(offset - lookback)
json_response = JSON.parse(response)
end
end

2
src/invidious/routes/api/v1/search.cr
View File

@ -32,7 +32,7 @@ module Invidious::Routes::API::V1::Search
return error_json(400, ex)
end
count, search_results = search(query, search_params, region).as(Tuple)
search_results = search(query, search_params, region)
JSON.build do |json|
json.array do
search_results.each do |item|

12
src/invidious/routes/api/v1/videos.cr
View File

@ -20,8 +20,6 @@ module Invidious::Routes::API::V1::Videos
end
def self.captions(env)
locale = env.get("preferences").as(Preferences).locale
env.response.content_type = "application/json"
id = env.params.url["id"]
@ -73,9 +71,9 @@ module Invidious::Routes::API::V1::Videos
env.response.content_type = "text/vtt; charset=UTF-8"
if lang
caption = captions.select { |caption| caption.language_code == lang }
caption = captions.select(&.language_code.== lang)
else
caption = captions.select { |caption| caption.name == label }
caption = captions.select(&.name.== label)
end
if caption.empty?
@ -149,8 +147,6 @@ module Invidious::Routes::API::V1::Videos
# thumbnails for individual scenes in a video.
# See https://support.jwplayer.com/articles/how-to-add-preview-thumbnails
def self.storyboards(env)
locale = env.get("preferences").as(Preferences).locale
env.response.content_type = "application/json"
id = env.params.url["id"]
@ -183,7 +179,7 @@ module Invidious::Routes::API::V1::Videos
env.response.content_type = "text/vtt"
storyboard = storyboards.select { |storyboard| width == "#{storyboard[:width]}" || height == "#{storyboard[:height]}" }
storyboard = storyboards.select { |sb| width == "#{sb[:width]}" || height == "#{sb[:height]}" }
if storyboard.empty?
haltf env, 404
@ -223,8 +219,6 @@ module Invidious::Routes::API::V1::Videos
end
def self.annotations(env)
locale = env.get("preferences").as(Preferences).locale
env.response.content_type = "text/xml"
id = env.params.url["id"]

11
src/invidious/routes/embed.cr
View File

@ -2,13 +2,11 @@
module Invidious::Routes::Embed
def self.redirect(env)
locale = env.get("preferences").as(Preferences).locale
if plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "")
begin
playlist = get_playlist(plid, locale: locale)
playlist = get_playlist(plid)
offset = env.params.query["index"]?.try &.to_i? || 0
videos = get_playlist_videos(playlist, offset: offset, locale: locale)
videos = get_playlist_videos(playlist, offset: offset)
rescue ex
return error_template(500, ex)
end
@ -26,7 +24,6 @@ module Invidious::Routes::Embed
end
def self.show(env)
locale = env.get("preferences").as(Preferences).locale
id = env.params.url["id"]
plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "")
@ -60,9 +57,9 @@ module Invidious::Routes::Embed
if plid
begin
playlist = get_playlist(plid, locale: locale)
playlist = get_playlist(plid)
offset = env.params.query["index"]?.try &.to_i? || 0
videos = get_playlist_videos(playlist, offset: offset, locale: locale)
videos = get_playlist_videos(playlist, offset: offset)
rescue ex
return error_template(500, ex)
end

2
src/invidious/routes/feeds.cr
View File

@ -265,7 +265,7 @@ module Invidious::Routes::Feeds
if plid.starts_with? "IV"
if playlist = Invidious::Database::Playlists.select(id: plid)
videos = get_playlist_videos(playlist, offset: 0, locale: locale)
videos = get_playlist_videos(playlist, offset: 0)
return XML.build(indent: " ", encoding: "UTF-8") do |xml|
xml.element("feed", "xmlns:yt": "http://www.youtube.com/xml/schemas/2015",

4
src/invidious/routes/login.cr
View File

@ -425,9 +425,9 @@ module Invidious::Routes::Login
found_valid_captcha = false
error_exception = Exception.new
tokens.each do |token|
tokens.each do |tok|
begin
validate_request(token, answer, env.request, HMAC_KEY, locale)
validate_request(tok, answer, env.request, HMAC_KEY, locale)
found_valid_captcha = true
rescue ex
error_exception = ex

14
src/invidious/routes/playlists.cr
View File

@ -66,7 +66,7 @@ module Invidious::Routes::Playlists
user = user.as(User)
playlist_id = env.params.query["list"]
playlist = get_playlist(playlist_id, locale)
playlist = get_playlist(playlist_id)
subscribe_playlist(user, playlist)
env.redirect "/playlist?list=#{playlist.id}"
@ -157,7 +157,7 @@ module Invidious::Routes::Playlists
end
begin
videos = get_playlist_videos(playlist, offset: (page - 1) * 100, locale: locale)
videos = get_playlist_videos(playlist, offset: (page - 1) * 100)
rescue ex
videos = [] of PlaylistVideo
end
@ -239,15 +239,13 @@ module Invidious::Routes::Playlists
query = env.params.query["q"]?
if query
begin
search_query, count, items, operators = process_search_query(query, page, user, region: nil)
search_query, items, operators = process_search_query(query, page, user, region: nil)
videos = items.select(SearchVideo).map(&.as(SearchVideo))
rescue ex
videos = [] of SearchVideo
count = 0
end
else
videos = [] of SearchVideo
count = 0
end
env.set "add_playlist_items", plid
@ -306,7 +304,7 @@ module Invidious::Routes::Playlists
begin
playlist_id = env.params.query["playlist_id"]
playlist = get_playlist(playlist_id, locale).as(InvidiousPlaylist)
playlist = get_playlist(playlist_id).as(InvidiousPlaylist)
raise "Invalid user" if playlist.author != user.email
rescue ex
if redirect
@ -397,7 +395,7 @@ module Invidious::Routes::Playlists
end
begin
playlist = get_playlist(plid, locale)
playlist = get_playlist(plid)
rescue ex
return error_template(500, ex)
end
@ -414,7 +412,7 @@ module Invidious::Routes::Playlists
end
begin
videos = get_playlist_videos(playlist, offset: (page - 1) * 100, locale: locale)
videos = get_playlist_videos(playlist, offset: (page - 1) * 100)
rescue ex
return error_template(500, "Error encountered while retrieving playlist videos.<br>#{ex.message}")
end

2
src/invidious/routes/search.cr
View File

@ -54,7 +54,7 @@ module Invidious::Routes::Search
user = env.get? "user"
begin
search_query, count, videos, operators = process_search_query(query, page, user, region: region)
search_query, videos, operators = process_search_query(query, page, user, region: region)
rescue ex : ChannelSearchException
return error_template(404, "Unable to find channel with id of '#{HTML.escape(ex.channel)}'. Are you sure that's an actual channel id? It should look like 'UC4QobU6STFB0P71PMvOGN5A'.")
rescue ex

24
src/invidious/routes/video_playback.cr
View File

@ -75,8 +75,8 @@ module Invidious::Routes::VideoPlayback
end
begin
client.get(url, headers) do |response|
response.headers.each do |key, value|
client.get(url, headers) do |resp|
resp.headers.each do |key, value|
if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
env.response.headers[key] = value
end
@ -84,7 +84,7 @@ module Invidious::Routes::VideoPlayback
env.response.headers["Access-Control-Allow-Origin"] = "*"
if location = response.headers["Location"]?
if location = resp.headers["Location"]?
location = URI.parse(location)
location = "#{location.request_target}&host=#{location.host}"
@ -95,7 +95,7 @@ module Invidious::Routes::VideoPlayback
return env.redirect location
end
IO.copy(response.body_io, env.response)
IO.copy(resp.body_io, env.response)
end
rescue ex
end
@ -132,15 +132,15 @@ module Invidious::Routes::VideoPlayback
headers["Range"] = "bytes=#{chunk_start}-#{chunk_end}"
begin
client.get(url, headers) do |response|
client.get(url, headers) do |resp|
if first_chunk
if !env.request.headers["Range"]? && response.status_code == 206
if !env.request.headers["Range"]? && resp.status_code == 206
env.response.status_code = 200
else
env.response.status_code = response.status_code
env.response.status_code = resp.status_code
end
response.headers.each do |key, value|
resp.headers.each do |key, value|
if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) && key.downcase != "content-range"
env.response.headers[key] = value
end
@ -148,7 +148,7 @@ module Invidious::Routes::VideoPlayback
env.response.headers["Access-Control-Allow-Origin"] = "*"
if location = response.headers["Location"]?
if location = resp.headers["Location"]?
location = URI.parse(location)
location = "#{location.request_target}&host=#{location.host}#{region ? "&region=#{region}" : ""}"
@ -161,8 +161,8 @@ module Invidious::Routes::VideoPlayback
env.response.headers["Content-Disposition"] = "attachment; filename=\"#{URI.encode_www_form(title)}\"; filename*=UTF-8''#{URI.encode_www_form(title)}"
end
if !response.headers.includes_word?("Transfer-Encoding", "chunked")
content_length = response.headers["Content-Range"].split("/")[-1].to_i64
if !resp.headers.includes_word?("Transfer-Encoding", "chunked")
content_length = resp.headers["Content-Range"].split("/")[-1].to_i64
if env.request.headers["Range"]?
env.response.headers["Content-Range"] = "bytes #{range_start}-#{range_end || (content_length - 1)}/#{content_length}"
env.response.content_length = ((range_end.try &.+ 1) || content_length) - range_start
@ -172,7 +172,7 @@ module Invidious::Routes::VideoPlayback
end
end
proxy_file(response, env)
proxy_file(resp, env)
end
rescue ex
if ex.message != "Error reading socket: Connection reset by peer"

28
src/invidious/search.cr
View File

@ -5,7 +5,7 @@ class ChannelSearchException < InfoException
end
end
def channel_search(query, page, channel)
def channel_search(query, page, channel) : Array(SearchItem)
response = YT_POOL.client &.get("/channel/#{channel}")
if response.status_code == 404
@ -24,25 +24,23 @@ def channel_search(query, page, channel)
continuation_items = response_json["onResponseReceivedActions"]?
.try &.[0]["appendContinuationItemsAction"]["continuationItems"]
return 0, [] of SearchItem if !continuation_items
return [] of SearchItem if !continuation_items
items = [] of SearchItem
continuation_items.as_a.select(&.as_h.has_key?("itemSectionRenderer")).each { |item|
extract_item(item["itemSectionRenderer"]["contents"].as_a[0])
.try { |t| items << t }
}
continuation_items.as_a.select(&.as_h.has_key?("itemSectionRenderer")).each do |item|
extract_item(item["itemSectionRenderer"]["contents"].as_a[0]).try { |t| items << t }
end
return items.size, items
return items
end
def search(query, search_params = produce_search_params(content_type: "all"), region = nil)
return 0, [] of SearchItem if query.empty?
def search(query, search_params = produce_search_params(content_type: "all"), region = nil) : Array(SearchItem)
return [] of SearchItem if query.empty?
client_config = YoutubeAPI::ClientConfig.new(region: region)
initial_data = YoutubeAPI.search(query, search_params, client_config: client_config)
items = extract_items(initial_data)
return items.size, items
return extract_items(initial_data)
end
def produce_search_params(page = 1, sort : String = "relevance", date : String = "", content_type : String = "",
@ -217,7 +215,7 @@ def process_search_query(query, page, user, region)
search_query = (query.split(" ") - operators).join(" ")
if channel
count, items = channel_search(search_query, page, channel)
items = channel_search(search_query, page, channel)
elsif subscriptions
if view_name
items = PG_DB.query_all("SELECT id,title,published,updated,ucid,author,length_seconds FROM (
@ -227,16 +225,14 @@ def process_search_query(query, page, user, region)
as document
FROM #{view_name}
) v_search WHERE v_search.document @@ plainto_tsquery($1) LIMIT 20 OFFSET $2;", search_query, (page - 1) * 20, as: ChannelVideo)
count = items.size
else
items = [] of ChannelVideo
count = 0
end
else
search_params = produce_search_params(page: page, sort: sort, date: date, content_type: content_type,
duration: duration, features: features)
count, items = search(search_query, search_params, region).as(Tuple)
items = search(search_query, search_params, region)
end
# Light processing to flatten search results out of Categories.
@ -254,5 +250,5 @@ def process_search_query(query, page, user, region)
end
end
{search_query, items_without_category.size, items_without_category, operators}
{search_query, items_without_category, operators}
end

12
src/invidious/users.cr
View File

@ -65,7 +65,6 @@ def fetch_user(sid, headers)
feed = YT_POOL.client &.get("/subscription_manager?disable_polymer=1", headers)
feed = XML.parse_html(feed.body)
channels = [] of String
channels = feed.xpath_nodes(%q(//ul[@id="guide-channels"]/li/a)).compact_map do |channel|
if {"Popular on YouTube", "Music", "Sports", "Gaming"}.includes? channel["title"]
nil
@ -157,12 +156,11 @@ def generate_captcha(key)
</svg>
END_SVG
image = ""
convert = Process.run(%(rsvg-convert -w 400 -h 400 -b none -f png), shell: true,
input: IO::Memory.new(clock_svg), output: Process::Redirect::Pipe) do |proc|
image = proc.output.gets_to_end
image = Base64.strict_encode(image)
image = "data:image/png;base64,#{image}"
image = "data:image/png;base64,"
image += Process.run(%(rsvg-convert -w 400 -h 400 -b none -f png), shell: true,
input: IO::Memory.new(clock_svg), output: Process::Redirect::Pipe
) do |proc|
Base64.strict_encode(proc.output.gets_to_end)
end
answer = "#{hour}:#{minute.to_s.rjust(2, '0')}:#{second.to_s.rjust(2, '0')}"

47
src/invidious/videos.cr
View File

@ -497,7 +497,7 @@ struct Video
end
def length_seconds : Int32
info["microformat"]?.try &.["playerMicroformatRenderer"]?.try &.["lengthSeconds"]?.try &.as_s.to_i ||
info.dig?("microformat", "playerMicroformatRenderer", "lengthSeconds").try &.as_s.to_i ||
info["videoDetails"]["lengthSeconds"]?.try &.as_s.to_i || 0
end
@ -519,7 +519,9 @@ struct Video
end
def published : Time
info["microformat"]?.try &.["playerMicroformatRenderer"]?.try &.["publishDate"]?.try { |t| Time.parse(t.as_s, "%Y-%m-%d", Time::Location::UTC) } || Time.utc
info
.dig?("microformat", "playerMicroformatRenderer", "publishDate")
.try { |t| Time.parse(t.as_s, "%Y-%m-%d", Time::Location::UTC) } || Time.utc
end
def published=(other : Time)
@ -545,8 +547,9 @@ struct Video
end
def premiere_timestamp : Time?
info["microformat"]?.try &.["playerMicroformatRenderer"]?
.try &.["liveBroadcastDetails"]?.try &.["startTimestamp"]?.try { |t| Time.parse_rfc3339(t.as_s) }
info
.dig?("microformat", "playerMicroformatRenderer", "liveBroadcastDetails", "startTimestamp")
.try { |t| Time.parse_rfc3339(t.as_s) }
end
def keywords
@ -558,8 +561,9 @@ struct Video
end
def allowed_regions
info["microformat"]?.try &.["playerMicroformatRenderer"]?
.try &.["availableCountries"]?.try &.as_a.map &.as_s || [] of String
info
.dig("microformat", "playerMicroformatRenderer", "availableCountries")
.try &.as_a.map &.as_s || [] of String
end
def author_thumbnail : String
@ -621,18 +625,11 @@ struct Video
end
def storyboards
storyboards = info["storyboards"]?
.try &.as_h
.try &.["playerStoryboardSpecRenderer"]?
.try &.["spec"]?
storyboards = info.dig?("storyboards", "playerStoryboardSpecRenderer", "spec")
.try &.as_s.split("|")
if !storyboards
if storyboard = info["storyboards"]?
.try &.as_h
.try &.["playerLiveStoryboardSpecRenderer"]?
.try &.["spec"]?
.try &.as_s
if storyboard = info.dig?("storyboards", "playerLiveStoryboardSpecRenderer", "spec").try &.as_s
return [{
url: storyboard.split("#")[0],
width: 106,
@ -661,8 +658,8 @@ struct Video
url = URI.parse(storyboards.shift)
params = HTTP::Params.parse(url.query || "")
storyboards.each_with_index do |storyboard, i|
width, height, count, storyboard_width, storyboard_height, interval, _, sigh = storyboard.split("#")
storyboards.each_with_index do |sb, i|
width, height, count, storyboard_width, storyboard_height, interval, _, sigh = sb.split("#")
params["sigh"] = sigh
url.query = params.to_s
@ -690,9 +687,8 @@ struct Video
end
def paid
reason = info["playabilityStatus"]?.try &.["reason"]?
paid = reason == "This video requires payment to watch." ? true : false
paid
reason = info.dig?("playabilityStatus", "reason").try &.as_s || ""
return reason.includes? "requires payment"
end
def premium
@ -716,8 +712,9 @@ struct Video
end
def description
description = info["microformat"]?.try &.["playerMicroformatRenderer"]?
.try &.["description"]?.try &.["simpleText"]?.try &.as_s || ""
description = info
.dig?("microformat", "playerMicroformatRenderer", "description", "simpleText")
.try &.as_s || ""
end
# TODO
@ -738,11 +735,11 @@ struct Video
end
def hls_manifest_url : String?
info["streamingData"]?.try &.["hlsManifestUrl"]?.try &.as_s
info.dig?("streamingData", "hlsManifestUrl").try &.as_s
end
def dash_manifest_url
info["streamingData"]?.try &.["dashManifestUrl"]?.try &.as_s
info.dig?("streamingData", "dashManifestUrl").try &.as_s
end
def genre : String
@ -758,7 +755,7 @@ struct Video
end
def is_family_friendly : Bool
info["microformat"]?.try &.["playerMicroformatRenderer"]["isFamilySafe"]?.try &.as_bool || false
info.dig?("microformat", "playerMicroformatRenderer", "isFamilySafe").try &.as_bool || false
end
def is_vr : Bool?

2
src/invidious/views/add_playlist_items.ecr
View File

@ -48,7 +48,7 @@
</div>
<div class="pure-u-1 pure-u-lg-3-5"></div>
<div class="pure-u-1 pure-u-lg-1-5" style="text-align:right">
<% if count >= 20 %>
<% if videos.size >= 20 %>
<a href="/add_playlist_items?list=<%= plid %>&q=<%= URI.encode_www_form(query.not_nil!) %>&page=<%= page + 1 %>">
<%= translate(locale, "Next page") %>
</a>

8
src/invidious/views/search.ecr
View File

@ -5,7 +5,7 @@
<% search_query_encoded = env.get?("search").try { |x| URI.encode_www_form(x.as(String), space_to_plus: true) } %>
<!-- Search redirection and filtering UI -->
<% if count == 0 %>
<% if videos.size == 0 %>
<h3 style="text-align: center">
<a href="/redirect?referer=<%= env.get?("current_page") %>"><%= translate(locale, "Broken? Try another Invidious Instance!") %></a>
</h3>
@ -98,7 +98,7 @@
</details>
<% end %>
<% if count == 0 %>
<% if videos.size == 0 %>
<hr style="margin: 0;"/>
<% else %>
<hr/>
@ -114,7 +114,7 @@
</div>
<div class="pure-u-1 pure-u-lg-3-5"></div>
<div class="pure-u-1 pure-u-lg-1-5" style="text-align:right">
<% if count >= 20 %>
<% if videos.size >= 20 %>
<a href="/search?q=<%= search_query_encoded %>&page=<%= page + 1 %>">
<%= translate(locale, "Next page") %>
</a>
@ -138,7 +138,7 @@
</div>
<div class="pure-u-1 pure-u-lg-3-5"></div>
<div class="pure-u-1 pure-u-lg-1-5" style="text-align:right">
<% if count >= 20 %>
<% if videos.size >= 20 %>
<a href="/search?q=<%= search_query_encoded %>&page=<%= page + 1 %>">
<%= translate(locale, "Next page") %>
</a>

5
src/invidious/yt_backend/youtube_api.cr
View File

@ -416,10 +416,9 @@ module YoutubeAPI
# Send the POST request
if {{ !flag?(:disable_quic) }} && CONFIG.use_quic
# Using QUIC client
response = YT_POOL.client(client_config.proxy_region,
body = YT_POOL.client(client_config.proxy_region,
&.post(url, headers: headers, body: data.to_json)
)
body = response.body
).body
else
# Using HTTP client
body = YT_POOL.client(client_config.proxy_region) do |client|