lug/invidious
7
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Don't overwrite published date for channel_videos

Browse Source
This commit is contained in:
Omar Roth 2019-04-20 10:18:54 -05:00
parent fb7068d415
commit 14620c32aa
1 changed files with 55 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

114
src/invidious/channels.cr
View File

@ -102,76 +102,72 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
auto_generated = true auto_generated = true
end end
if !pull_all_videos page = 1
url = produce_channel_videos_url(ucid, 1, auto_generated: auto_generated)
response = client.get(url)
json = JSON.parse(response.body)
if json["content_html"]? && !json["content_html"].as_s.empty? url = produce_channel_videos_url(ucid, page, auto_generated: auto_generated)
document = XML.parse_html(json["content_html"].as_s) response = client.get(url)
nodeset = document.xpath_nodes(%q(//li[contains(@class, "feed-item-container")])) json = JSON.parse(response.body)
if auto_generated if json["content_html"]? && !json["content_html"].as_s.empty?
videos = extract_videos(nodeset) document = XML.parse_html(json["content_html"].as_s)
else nodeset = document.xpath_nodes(%q(//li[contains(@class, "feed-item-container")]))
videos = extract_videos(nodeset, ucid, author)
end if auto_generated
videos = extract_videos(nodeset)
else
videos = extract_videos(nodeset, ucid, author)
end end
end
videos ||= [] of ChannelVideo videos ||= [] of ChannelVideo
rss.xpath_nodes("//feed/entry").each do |entry| rss.xpath_nodes("//feed/entry").each do |entry|
video_id = entry.xpath_node("videoid").not_nil!.content video_id = entry.xpath_node("videoid").not_nil!.content
title = entry.xpath_node("title").not_nil!.content title = entry.xpath_node("title").not_nil!.content
published = Time.parse_rfc3339(entry.xpath_node("published").not_nil!.content) published = Time.parse_rfc3339(entry.xpath_node("published").not_nil!.content)
updated = Time.parse_rfc3339(entry.xpath_node("updated").not_nil!.content) updated = Time.parse_rfc3339(entry.xpath_node("updated").not_nil!.content)
author = entry.xpath_node("author/name").not_nil!.content author = entry.xpath_node("author/name").not_nil!.content
ucid = entry.xpath_node("channelid").not_nil!.content ucid = entry.xpath_node("channelid").not_nil!.content
channel_video = videos.select { |video| video.id == video_id }[0]? channel_video = videos.select { |video| video.id == video_id }[0]?
length_seconds = channel_video.try &.length_seconds length_seconds = channel_video.try &.length_seconds
length_seconds ||= 0 length_seconds ||= 0
live_now = channel_video.try &.live_now live_now = channel_video.try &.live_now
live_now ||= false live_now ||= false
premiere_timestamp = channel_video.try &.premiere_timestamp premiere_timestamp = channel_video.try &.premiere_timestamp
# Deliver notifications to `/api/v1/auth/notifications` video = ChannelVideo.new(
# payload = { id: video_id,
# "key" => video_id, title: title,
# "topic" => ucid, published: published,
# }.to_json updated: Time.now,
# PG_DB.exec("NOTIFY notifications, E'#{payload}'") ucid: ucid,
author: author,
length_seconds: length_seconds,
live_now: live_now,
premiere_timestamp: premiere_timestamp
)
video = ChannelVideo.new( db.exec("UPDATE users SET notifications = notifications || $1 \
id: video_id,
title: title,
published: published,
updated: Time.now,
ucid: ucid,
author: author,
length_seconds: length_seconds,
live_now: live_now,
premiere_timestamp: premiere_timestamp
)
db.exec("UPDATE users SET notifications = notifications || $1 \
WHERE updated < $2 AND $3 = ANY(subscriptions) AND $1 <> ALL(notifications)", video.id, video.published, ucid) WHERE updated < $2 AND $3 = ANY(subscriptions) AND $1 <> ALL(notifications)", video.id, video.published, ucid)
video_array = video.to_a video_array = video.to_a
args = arg_array(video_array) args = arg_array(video_array)
# We don't include the 'premire_timestamp' here because channel pages don't include them, # We don't include the 'premire_timestamp' here because channel pages don't include them,
# meaning the above timestamp is always null # meaning the above timestamp is always null
db.exec("INSERT INTO channel_videos VALUES (#{args}) \ db.exec("INSERT INTO channel_videos VALUES (#{args}) \
ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, \ ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, \
updated = $4, ucid = $5, author = $6, length_seconds = $7, \ updated = $4, ucid = $5, author = $6, length_seconds = $7, \
live_now = $8", video_array) live_now = $8", video_array)
end end
else
page = 1 if pull_all_videos
page += 1
ids = [] of String ids = [] of String
loop do loop do
@ -186,6 +182,8 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
break break
end end
nodeset = nodeset.not_nil!
if auto_generated if auto_generated
videos = extract_videos(nodeset) videos = extract_videos(nodeset)
else else
@ -216,16 +214,14 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
video_array = video.to_a video_array = video.to_a
args = arg_array(video_array) args = arg_array(video_array)
# We don't include the 'premire_timestamp' here because channel pages don't include them, # We don't update the 'premire_timestamp' here because channel pages don't include them
# meaning the above timestamp is always null
db.exec("INSERT INTO channel_videos VALUES (#{args}) \ db.exec("INSERT INTO channel_videos VALUES (#{args}) \
ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, \ ON CONFLICT (id) DO UPDATE SET title = $2, updated = $4, \
updated = $4, ucid = $5, author = $6, length_seconds = $7, \ ucid = $5, author = $6, length_seconds = $7, live_now = $8", video_array)
live_now = $8", video_array)
end end
end end
if count < 30 if count < 25
break break
end end