From b0df3774dbb9494007bd5a7de62ae8cf756f3c7c Mon Sep 17 00:00:00 2001 From: src-tinkerer Date: Wed, 1 Nov 2023 21:56:25 +0330 Subject: [PATCH 01/29] Add sort options to streams --- src/invidious/channels/videos.cr | 89 ++++++++++++++++++++++--- src/invidious/routes/api/v1/channels.cr | 3 +- src/invidious/routes/channels.cr | 7 +- 3 files changed, 86 insertions(+), 13 deletions(-) diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr index beb86e08..87af0caf 100644 --- a/src/invidious/channels/videos.cr +++ b/src/invidious/channels/videos.cr @@ -68,6 +68,76 @@ def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = " return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" end +def produce_channel_livestreams_continuation(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) + object_inner_2 = { + "2:0:embedded" => { + "1:0:varint" => 0_i64, + }, + "5:varint" => 50_i64, + "6:varint" => 1_i64, + "7:varint" => (page * 30).to_i64, + "9:varint" => 1_i64, + "10:varint" => 0_i64, + } + + object_inner_2_encoded = object_inner_2 + .try { |i| Protodec::Any.cast_json(i) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + .try { |i| URI.encode_www_form(i) } + + sort_by_numerical = + case sort_by + when "newest" then 1_i64 + when "popular" then 2_i64 + when "oldest" then 4_i64 + else 1_i64 # Fallback to "newest" + end + + object_inner_1 = { + "110:embedded" => { + "3:embedded" => { + "14:embedded" => { + "1:embedded" => { + "1:string" => object_inner_2_encoded, + }, + "2:embedded" => { + "1:string" => "00000000-0000-0000-0000-000000000000", + }, + "3:varint" => sort_by_numerical, + }, + }, + }, + } + + object_inner_1_encoded = object_inner_1 + .try { |i| Protodec::Any.cast_json(i) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + .try { |i| URI.encode_www_form(i) } + + object = { + "80226972:embedded" => { + "2:string" => ucid, + "3:string" => object_inner_1_encoded, + "35:string" => "browse-feed#{ucid}videos102", + }, + } + + continuation = object.try { |i| Protodec::Any.cast_json(i) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + .try { |i| URI.encode_www_form(i) } + + return continuation +end + +# Used in bypass_captcha_job.cr +def produce_channel_livestream_url(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) + continuation = produce_channel_livestreams_continuation(ucid, page, auto_generated, sort_by, v2) + return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" +end + module Invidious::Channel::Tabs extend self @@ -144,21 +214,24 @@ module Invidious::Channel::Tabs # Livestreams # ------------------- - def get_livestreams(channel : AboutChannel, continuation : String? = nil) + def make_initial_livestream_ctoken(ucid, sort_by) : String + return produce_channel_livestreams_continuation(ucid, sort_by: sort_by) + end + + def get_livestreams(channel : AboutChannel, continuation : String? = nil, sort_by = "newest") if continuation.nil? - # EgdzdHJlYW1z8gYECgJ6AA%3D%3D is the protobuf object to load "streams" - initial_data = YoutubeAPI.browse(channel.ucid, params: "EgdzdHJlYW1z8gYECgJ6AA%3D%3D") - else - initial_data = YoutubeAPI.browse(continuation: continuation) + continuation ||= make_initial_livestream_ctoken(channel.ucid, sort_by) end + initial_data = YoutubeAPI.browse(continuation: continuation) + return extract_items(initial_data, channel.author, channel.ucid) end - def get_60_livestreams(channel : AboutChannel, continuation : String? = nil) + def get_60_livestreams(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest") if continuation.nil? - # Fetch the first "page" of streams - items, next_continuation = get_livestreams(channel) + # Fetch the first "page" of stream + items, next_continuation = get_livestreams(channel, sort_by: sort_by) else # Fetch a "page" of streams using the given continuation token items, next_continuation = get_livestreams(channel, continuation: continuation) diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr index 67018660..f759f6ab 100644 --- a/src/invidious/routes/api/v1/channels.cr +++ b/src/invidious/routes/api/v1/channels.cr @@ -207,11 +207,12 @@ module Invidious::Routes::API::V1::Channels get_channel() # Retrieve continuation from URL parameters + sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" continuation = env.params.query["continuation"]? begin videos, next_continuation = Channel::Tabs.get_60_livestreams( - channel, continuation: continuation + channel, continuation: continuation, sort_by: sort_by ) rescue ex return error_json(500, ex) diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr index d4d8b1c1..c18644ec 100644 --- a/src/invidious/routes/channels.cr +++ b/src/invidious/routes/channels.cr @@ -81,13 +81,12 @@ module Invidious::Routes::Channels return env.redirect "/channel/#{channel.ucid}" end - # TODO: support sort option for livestreams - sort_by = "" - sort_options = [] of String + sort_by = env.params.query["sort_by"]?.try &.downcase + sort_options = {"newest", "oldest", "popular"} # Fetch items and continuation token items, next_continuation = Channel::Tabs.get_60_livestreams( - channel, continuation: continuation + channel, continuation: continuation, sort_by: (sort_by || "newest") ) selected_tab = Frontend::ChannelPage::TabsAvailable::Streams From 63e5d72466d8b879bfa54f60ed475451e868e691 Mon Sep 17 00:00:00 2001 From: src-tinkerer Date: Mon, 20 Nov 2023 15:50:59 +0330 Subject: [PATCH 02/29] Remove unused function produce_channel_livestream_url --- src/invidious/channels/videos.cr | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr index 87af0caf..c389dec3 100644 --- a/src/invidious/channels/videos.cr +++ b/src/invidious/channels/videos.cr @@ -132,12 +132,6 @@ def produce_channel_livestreams_continuation(ucid, page = 1, auto_generated = ni return continuation end -# Used in bypass_captcha_job.cr -def produce_channel_livestream_url(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) - continuation = produce_channel_livestreams_continuation(ucid, page, auto_generated, sort_by, v2) - return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" -end - module Invidious::Channel::Tabs extend self From 0d63ad5a7f13316652117205b1efe2ff2bda00e6 Mon Sep 17 00:00:00 2001 From: src-tinkerer Date: Wed, 22 Nov 2023 14:50:08 +0330 Subject: [PATCH 03/29] Use a single function for fetching channel contents --- src/invidious/channels/videos.cr | 93 ++++++-------------------------- 1 file changed, 16 insertions(+), 77 deletions(-) diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr index c389dec3..368ec2b8 100644 --- a/src/invidious/channels/videos.cr +++ b/src/invidious/channels/videos.cr @@ -1,4 +1,4 @@ -def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) +def produce_channel_content_continuation(ucid, content, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) object_inner_2 = { "2:0:embedded" => { "1:0:varint" => 0_i64, @@ -16,6 +16,13 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so .try { |i| Base64.urlsafe_encode(i) } .try { |i| URI.encode_www_form(i) } + content_numerical = + case content + when "videos" then 15 + when "livestreams" then 14 + else 15 # Fallback to "videos" + end + sort_by_numerical = case sort_by when "newest" then 1_i64 @@ -27,7 +34,7 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so object_inner_1 = { "110:embedded" => { "3:embedded" => { - "15:embedded" => { + "#{content_numerical}:embedded" => { "1:embedded" => { "1:string" => object_inner_2_encoded, }, @@ -62,76 +69,16 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so return continuation end +def make_initial_content_ctoken(ucid, content, sort_by) : String + return produce_channel_content_continuation(ucid, content, sort_by: sort_by) +end + # Used in bypass_captcha_job.cr def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) - continuation = produce_channel_videos_continuation(ucid, page, auto_generated, sort_by, v2) + continuation = produce_channel_content_continuation(ucid, "videos", page, auto_generated, sort_by, v2) return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en" end -def produce_channel_livestreams_continuation(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) - object_inner_2 = { - "2:0:embedded" => { - "1:0:varint" => 0_i64, - }, - "5:varint" => 50_i64, - "6:varint" => 1_i64, - "7:varint" => (page * 30).to_i64, - "9:varint" => 1_i64, - "10:varint" => 0_i64, - } - - object_inner_2_encoded = object_inner_2 - .try { |i| Protodec::Any.cast_json(i) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } - - sort_by_numerical = - case sort_by - when "newest" then 1_i64 - when "popular" then 2_i64 - when "oldest" then 4_i64 - else 1_i64 # Fallback to "newest" - end - - object_inner_1 = { - "110:embedded" => { - "3:embedded" => { - "14:embedded" => { - "1:embedded" => { - "1:string" => object_inner_2_encoded, - }, - "2:embedded" => { - "1:string" => "00000000-0000-0000-0000-000000000000", - }, - "3:varint" => sort_by_numerical, - }, - }, - }, - } - - object_inner_1_encoded = object_inner_1 - .try { |i| Protodec::Any.cast_json(i) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } - - object = { - "80226972:embedded" => { - "2:string" => ucid, - "3:string" => object_inner_1_encoded, - "35:string" => "browse-feed#{ucid}videos102", - }, - } - - continuation = object.try { |i| Protodec::Any.cast_json(i) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } - - return continuation -end - module Invidious::Channel::Tabs extend self @@ -139,10 +86,6 @@ module Invidious::Channel::Tabs # Regular videos # ------------------- - def make_initial_video_ctoken(ucid, sort_by) : String - return produce_channel_videos_continuation(ucid, sort_by: sort_by) - end - # Wrapper for AboutChannel, as we still need to call get_videos with # an author name and ucid directly (e.g in RSS feeds). # TODO: figure out how to get rid of that @@ -164,7 +107,7 @@ module Invidious::Channel::Tabs end def get_videos(author : String, ucid : String, *, continuation : String? = nil, sort_by = "newest") - continuation ||= make_initial_video_ctoken(ucid, sort_by) + continuation ||= make_initial_content_ctoken(ucid, "videos", sort_by) initial_data = YoutubeAPI.browse(continuation: continuation) return extract_items(initial_data, author, ucid) @@ -208,13 +151,9 @@ module Invidious::Channel::Tabs # Livestreams # ------------------- - def make_initial_livestream_ctoken(ucid, sort_by) : String - return produce_channel_livestreams_continuation(ucid, sort_by: sort_by) - end - def get_livestreams(channel : AboutChannel, continuation : String? = nil, sort_by = "newest") if continuation.nil? - continuation ||= make_initial_livestream_ctoken(channel.ucid, sort_by) + continuation ||= make_initial_content_ctoken(channel.ucid, "livestreams", sort_by) end initial_data = YoutubeAPI.browse(continuation: continuation) From 162b89d9427ec4cbeb4661d7a7866847ba582880 Mon Sep 17 00:00:00 2001 From: src-tinkerer Date: Thu, 23 Nov 2023 14:44:37 +0330 Subject: [PATCH 04/29] Fix format in videos.cr --- src/invidious/channels/videos.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr index 368ec2b8..80ec498c 100644 --- a/src/invidious/channels/videos.cr +++ b/src/invidious/channels/videos.cr @@ -18,9 +18,9 @@ def produce_channel_content_continuation(ucid, content, page = 1, auto_generated content_numerical = case content - when "videos" then 15 + when "videos" then 15 when "livestreams" then 14 - else 15 # Fallback to "videos" + else 15 # Fallback to "videos" end sort_by_numerical = From 6251d8d43f8b0491e4ff86fa8433b92a91b5fafd Mon Sep 17 00:00:00 2001 From: src-tinkerer Date: Sat, 25 Nov 2023 00:46:11 +0330 Subject: [PATCH 05/29] Rename a variable in videos.cr --- src/invidious/channels/videos.cr | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr index 80ec498c..cc683788 100644 --- a/src/invidious/channels/videos.cr +++ b/src/invidious/channels/videos.cr @@ -1,4 +1,4 @@ -def produce_channel_content_continuation(ucid, content, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) +def produce_channel_content_continuation(ucid, content_type, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) object_inner_2 = { "2:0:embedded" => { "1:0:varint" => 0_i64, @@ -16,8 +16,8 @@ def produce_channel_content_continuation(ucid, content, page = 1, auto_generated .try { |i| Base64.urlsafe_encode(i) } .try { |i| URI.encode_www_form(i) } - content_numerical = - case content + content_type_numerical = + case content_type when "videos" then 15 when "livestreams" then 14 else 15 # Fallback to "videos" @@ -34,7 +34,7 @@ def produce_channel_content_continuation(ucid, content, page = 1, auto_generated object_inner_1 = { "110:embedded" => { "3:embedded" => { - "#{content_numerical}:embedded" => { + "#{content_type_numerical}:embedded" => { "1:embedded" => { "1:string" => object_inner_2_encoded, }, @@ -69,8 +69,8 @@ def produce_channel_content_continuation(ucid, content, page = 1, auto_generated return continuation end -def make_initial_content_ctoken(ucid, content, sort_by) : String - return produce_channel_content_continuation(ucid, content, sort_by: sort_by) +def make_initial_content_ctoken(ucid, content_type, sort_by) : String + return produce_channel_content_continuation(ucid, content_type, sort_by: sort_by) end # Used in bypass_captcha_job.cr From 5f2b43d6534fcbb12d9c66c4e99b16863bd29893 Mon Sep 17 00:00:00 2001 From: src-tinkerer Date: Sat, 25 Nov 2023 00:48:27 +0330 Subject: [PATCH 06/29] Remove unecessary if condition in videos.cr --- src/invidious/channels/videos.cr | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr index cc683788..02fc4d2f 100644 --- a/src/invidious/channels/videos.cr +++ b/src/invidious/channels/videos.cr @@ -152,9 +152,7 @@ module Invidious::Channel::Tabs # ------------------- def get_livestreams(channel : AboutChannel, continuation : String? = nil, sort_by = "newest") - if continuation.nil? - continuation ||= make_initial_content_ctoken(channel.ucid, "livestreams", sort_by) - end + continuation ||= make_initial_content_ctoken(channel.ucid, "livestreams", sort_by) initial_data = YoutubeAPI.browse(continuation: continuation) From 1363fb809436464de57b90113864ff50867a9dae Mon Sep 17 00:00:00 2001 From: Brahim Hadriche Date: Tue, 28 Nov 2023 21:34:17 -0500 Subject: [PATCH 07/29] Fix error code for disabled popular endpoint --- src/invidious/routes/api/v1/feeds.cr | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/invidious/routes/api/v1/feeds.cr b/src/invidious/routes/api/v1/feeds.cr index 41865f34..0ee22ca6 100644 --- a/src/invidious/routes/api/v1/feeds.cr +++ b/src/invidious/routes/api/v1/feeds.cr @@ -30,8 +30,7 @@ module Invidious::Routes::API::V1::Feeds env.response.content_type = "application/json" if !CONFIG.popular_enabled - error_message = {"error" => "Administrator has disabled this endpoint."}.to_json - haltf env, 400, error_message + return error_json(403, "Administrator has disabled this endpoint.") end JSON.build do |json| From cf61af67abc1bffdeefd080542d11c2eae13d754 Mon Sep 17 00:00:00 2001 From: src-tinkerer Date: Thu, 30 Nov 2023 14:34:01 +0330 Subject: [PATCH 08/29] Update src/invidious/routes/channels.cr sort_by for consistency --- src/invidious/routes/channels.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr index c18644ec..b5b4ad96 100644 --- a/src/invidious/routes/channels.cr +++ b/src/invidious/routes/channels.cr @@ -81,12 +81,12 @@ module Invidious::Routes::Channels return env.redirect "/channel/#{channel.ucid}" end - sort_by = env.params.query["sort_by"]?.try &.downcase + sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" sort_options = {"newest", "oldest", "popular"} # Fetch items and continuation token items, next_continuation = Channel::Tabs.get_60_livestreams( - channel, continuation: continuation, sort_by: (sort_by || "newest") + channel, continuation: continuation, sort_by: sort_by ) selected_tab = Frontend::ChannelPage::TabsAvailable::Streams From 4adb4c00d2099ad7892579bfe4777d6f64a807a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Wilczy=C5=84ski?= Date: Sat, 24 Feb 2024 20:01:16 +0100 Subject: [PATCH 09/29] routes: Allow embedding videos in local HTML files (fixes #4448) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The current Content Security Policy does not allow to embed videos inside local HTML files which are viewed in the browser via the file protocol. This commit adds the file protocol to the allowed frame ancestors, so that the embedded videos load correctly in local HTML files. This behaviour is consistent which how the official YouTube website allows to embed videos from itself. Signed-off-by: Tomasz Wilczyński --- src/invidious/routes/before_all.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/invidious/routes/before_all.cr b/src/invidious/routes/before_all.cr index 396840a4..5695dee9 100644 --- a/src/invidious/routes/before_all.cr +++ b/src/invidious/routes/before_all.cr @@ -30,7 +30,7 @@ module Invidious::Routes::BeforeAll # Only allow the pages at /embed/* to be embedded if env.request.resource.starts_with?("/embed") - frame_ancestors = "'self' http: https:" + frame_ancestors = "'self' file: http: https:" else frame_ancestors = "'none'" end From c5eb10b21f742041ad3dad809a0f5aa6ce339c17 Mon Sep 17 00:00:00 2001 From: Brahim Hadriche Date: Mon, 1 Apr 2024 10:02:49 -0400 Subject: [PATCH 10/29] Revert "Fix error code for disabled popular endpoint" This reverts commit 1363fb809436464de57b90113864ff50867a9dae. --- src/invidious/routes/api/v1/feeds.cr | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/invidious/routes/api/v1/feeds.cr b/src/invidious/routes/api/v1/feeds.cr index 0ee22ca6..41865f34 100644 --- a/src/invidious/routes/api/v1/feeds.cr +++ b/src/invidious/routes/api/v1/feeds.cr @@ -30,7 +30,8 @@ module Invidious::Routes::API::V1::Feeds env.response.content_type = "application/json" if !CONFIG.popular_enabled - return error_json(403, "Administrator has disabled this endpoint.") + error_message = {"error" => "Administrator has disabled this endpoint."}.to_json + haltf env, 400, error_message end JSON.build do |json| From b0c6bdf44c7fdff97d4fd408a7fede67f82e68a6 Mon Sep 17 00:00:00 2001 From: Brahim Hadriche Date: Mon, 1 Apr 2024 10:03:29 -0400 Subject: [PATCH 11/29] use 403 code --- src/invidious/routes/api/v1/feeds.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/invidious/routes/api/v1/feeds.cr b/src/invidious/routes/api/v1/feeds.cr index 41865f34..fea2993c 100644 --- a/src/invidious/routes/api/v1/feeds.cr +++ b/src/invidious/routes/api/v1/feeds.cr @@ -31,7 +31,7 @@ module Invidious::Routes::API::V1::Feeds if !CONFIG.popular_enabled error_message = {"error" => "Administrator has disabled this endpoint."}.to_json - haltf env, 400, error_message + haltf env, 403, error_message end JSON.build do |json| From b0ec359028c05367d3aa5b230a9cbf87de661191 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Sat, 27 Apr 2024 20:01:19 +0200 Subject: [PATCH 12/29] CI: Bump Crystal version matrix --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 057e4d61..08d2e16b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,10 +38,10 @@ jobs: matrix: stable: [true] crystal: - - 1.7.3 - - 1.8.2 - 1.9.2 - 1.10.1 + - 1.11.2 + - 1.12.1 include: - crystal: nightly stable: false From 470245de54f9a2941cdab15a55edb074f1a9897f Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Sat, 27 Apr 2024 20:48:42 +0200 Subject: [PATCH 13/29] YtAPI: Remove API keys like official clients --- src/invidious/yt_backend/youtube_api.cr | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index 727ce9a3..c8b037c8 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -5,9 +5,6 @@ module YoutubeAPI extend self - private DEFAULT_API_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8" - private ANDROID_API_KEY = "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w" - # For Android versions, see https://en.wikipedia.org/wiki/Android_version_history private ANDROID_APP_VERSION = "19.14.42" private ANDROID_USER_AGENT = "com.google.android.youtube/19.14.42 (Linux; U; Android 12; US) gzip" @@ -52,7 +49,6 @@ module YoutubeAPI name: "WEB", name_proto: "1", version: "2.20240304.00.00", - api_key: DEFAULT_API_KEY, screen: "WATCH_FULL_SCREEN", os_name: "Windows", os_version: WINDOWS_VERSION, @@ -62,7 +58,6 @@ module YoutubeAPI name: "WEB_EMBEDDED_PLAYER", name_proto: "56", version: "1.20240303.00.00", - api_key: DEFAULT_API_KEY, screen: "EMBED", os_name: "Windows", os_version: WINDOWS_VERSION, @@ -72,7 +67,6 @@ module YoutubeAPI name: "MWEB", name_proto: "2", version: "2.20240304.08.00", - api_key: DEFAULT_API_KEY, os_name: "Android", os_version: ANDROID_VERSION, platform: "MOBILE", @@ -81,7 +75,6 @@ module YoutubeAPI name: "WEB", name_proto: "1", version: "2.20240304.00.00", - api_key: DEFAULT_API_KEY, screen: "EMBED", os_name: "Windows", os_version: WINDOWS_VERSION, @@ -94,7 +87,6 @@ module YoutubeAPI name: "ANDROID", name_proto: "3", version: ANDROID_APP_VERSION, - api_key: ANDROID_API_KEY, android_sdk_version: ANDROID_SDK_VERSION, user_agent: ANDROID_USER_AGENT, os_name: "Android", @@ -105,13 +97,11 @@ module YoutubeAPI name: "ANDROID_EMBEDDED_PLAYER", name_proto: "55", version: ANDROID_APP_VERSION, - api_key: "AIzaSyCjc_pVEDi4qsv5MtC2dMXzpIaDoRFLsxw", }, ClientType::AndroidScreenEmbed => { name: "ANDROID", name_proto: "3", version: ANDROID_APP_VERSION, - api_key: DEFAULT_API_KEY, screen: "EMBED", android_sdk_version: ANDROID_SDK_VERSION, user_agent: ANDROID_USER_AGENT, @@ -123,7 +113,6 @@ module YoutubeAPI name: "ANDROID_TESTSUITE", name_proto: "30", version: ANDROID_TS_APP_VERSION, - api_key: ANDROID_API_KEY, android_sdk_version: ANDROID_SDK_VERSION, user_agent: ANDROID_TS_USER_AGENT, os_name: "Android", @@ -137,7 +126,6 @@ module YoutubeAPI name: "IOS", name_proto: "5", version: IOS_APP_VERSION, - api_key: "AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc", user_agent: IOS_USER_AGENT, device_make: "Apple", device_model: "iPhone14,5", @@ -149,7 +137,6 @@ module YoutubeAPI name: "IOS_MESSAGES_EXTENSION", name_proto: "66", version: IOS_APP_VERSION, - api_key: DEFAULT_API_KEY, user_agent: IOS_USER_AGENT, device_make: "Apple", device_model: "iPhone14,5", @@ -161,7 +148,6 @@ module YoutubeAPI name: "IOS_MUSIC", name_proto: "26", version: "6.42", - api_key: "AIzaSyBAETezhkwP0ZWA02RsqT1zu78Fpt0bC_s", user_agent: "com.google.ios.youtubemusic/6.42 (iPhone14,5; U; CPU iOS 17_4 like Mac OS X;)", device_make: "Apple", device_model: "iPhone14,5", @@ -176,13 +162,11 @@ module YoutubeAPI name: "TVHTML5", name_proto: "7", version: "7.20240304.10.00", - api_key: DEFAULT_API_KEY, }, ClientType::TvHtml5ScreenEmbed => { name: "TVHTML5_SIMPLY_EMBEDDED_PLAYER", name_proto: "85", version: "2.0", - api_key: DEFAULT_API_KEY, screen: "EMBED", }, } @@ -237,11 +221,6 @@ module YoutubeAPI HARDCODED_CLIENTS[@client_type][:version] end - # :ditto: - def api_key : String - HARDCODED_CLIENTS[@client_type][:api_key] - end - # :ditto: def screen : String HARDCODED_CLIENTS[@client_type][:screen]? || "" @@ -606,7 +585,7 @@ module YoutubeAPI client_config ||= DEFAULT_CLIENT_CONFIG # Query parameters - url = "#{endpoint}?key=#{client_config.api_key}&prettyPrint=false" + url = "#{endpoint}?prettyPrint=false" headers = HTTP::Headers{ "Content-Type" => "application/json; charset=UTF-8", From 2fdb6dd6441287e51abe06f4d9e6ce0586f916ff Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Sat, 27 Apr 2024 21:02:37 +0200 Subject: [PATCH 14/29] CI: Bump Crystal version in docker too --- .github/workflows/container-release.yml | 2 +- docker/Dockerfile | 2 +- docker/Dockerfile.arm64 | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/container-release.yml b/.github/workflows/container-release.yml index e44ac200..795995e9 100644 --- a/.github/workflows/container-release.yml +++ b/.github/workflows/container-release.yml @@ -26,7 +26,7 @@ jobs: - name: Install Crystal uses: crystal-lang/install-crystal@v1.8.0 with: - crystal: 1.9.2 + crystal: 1.12.1 - name: Run lint run: | diff --git a/docker/Dockerfile b/docker/Dockerfile index ace096bf..3d9323fd 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM crystallang/crystal:1.8.2-alpine AS builder +FROM crystallang/crystal:1.12.1-alpine AS builder RUN apk add --no-cache sqlite-static yaml-static diff --git a/docker/Dockerfile.arm64 b/docker/Dockerfile.arm64 index 602f3ab2..f054b326 100644 --- a/docker/Dockerfile.arm64 +++ b/docker/Dockerfile.arm64 @@ -1,5 +1,5 @@ -FROM alpine:3.18 AS builder -RUN apk add --no-cache 'crystal=1.8.2-r0' shards sqlite-static yaml-static yaml-dev libxml2-static zlib-static openssl-libs-static openssl-dev musl-dev xz-static +FROM alpine:3.19 AS builder +RUN apk add --no-cache 'crystal=1.10.1-r0' shards sqlite-static yaml-static yaml-dev libxml2-static zlib-static openssl-libs-static openssl-dev musl-dev xz-static ARG release From 9d66676f2dbb18a87ca7515e839f1c64688ecd39 Mon Sep 17 00:00:00 2001 From: Fijxu Date: Wed, 1 May 2024 22:17:41 -0400 Subject: [PATCH 15/29] Use full URL in the og:image property. --- src/invidious/views/channel.ecr | 4 ++-- src/invidious/views/watch.ecr | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/invidious/views/channel.ecr b/src/invidious/views/channel.ecr index 09df106d..a84e44bc 100644 --- a/src/invidious/views/channel.ecr +++ b/src/invidious/views/channel.ecr @@ -30,13 +30,13 @@ - + - + <%- end -%> diff --git a/src/invidious/views/watch.ecr b/src/invidious/views/watch.ecr index 7a1cf2c3..9e7467dd 100644 --- a/src/invidious/views/watch.ecr +++ b/src/invidious/views/watch.ecr @@ -10,7 +10,7 @@ - + From 0224162ad22dc19d58a73202d796eb3e99f0a71c Mon Sep 17 00:00:00 2001 From: syeopite Date: Tue, 11 Jun 2024 17:57:33 -0700 Subject: [PATCH 16/29] Rewrite transcript logic to be more generic The transcript logic in Invidious was written specifically as a workaround for captions, and not transcripts as a feature. This commit genericises the logic a bit as so it can be used for implementing transcripts within Invidious' API and UI as well. The most notable change is the added parsing of section headings when it was previously skipped over in favor of regular lines. --- src/invidious/routes/api/v1/videos.cr | 9 ++- src/invidious/videos/transcript.cr | 90 +++++++++++++++++---------- 2 files changed, 63 insertions(+), 36 deletions(-) diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr index 9281f4dd..faff2f59 100644 --- a/src/invidious/routes/api/v1/videos.cr +++ b/src/invidious/routes/api/v1/videos.cr @@ -89,9 +89,14 @@ module Invidious::Routes::API::V1::Videos if CONFIG.use_innertube_for_captions params = Invidious::Videos::Transcript.generate_param(id, caption.language_code, caption.auto_generated) - initial_data = YoutubeAPI.get_transcript(params) - webvtt = Invidious::Videos::Transcript.convert_transcripts_to_vtt(initial_data, caption.language_code) + transcript = Invidious::Videos::Transcript.from_raw( + YoutubeAPI.get_transcript(params), + caption.language_code, + caption.auto_generated + ) + + webvtt = transcript.to_vtt else # Timedtext API handling url = URI.parse("#{caption.base_url}&tlang=#{tlang}").request_target diff --git a/src/invidious/videos/transcript.cr b/src/invidious/videos/transcript.cr index dac00eea..42f29f46 100644 --- a/src/invidious/videos/transcript.cr +++ b/src/invidious/videos/transcript.cr @@ -1,8 +1,21 @@ module Invidious::Videos - # Namespace for methods primarily relating to Transcripts - module Transcript - record TranscriptLine, start_ms : Time::Span, end_ms : Time::Span, line : String + # A `Transcripts` struct encapsulates a sequence of lines that together forms the whole transcript for a given YouTube video. + # These lines can be categorized into two types: section headings and regular lines representing content from the video. + struct Transcript + # Types + record HeadingLine, start_ms : Time::Span, end_ms : Time::Span, line : String + record RegularLine, start_ms : Time::Span, end_ms : Time::Span, line : String + alias TranscriptLine = HeadingLine | RegularLine + property lines : Array(TranscriptLine) + property language_code : String + property auto_generated : Bool + + # Initializes a new Transcript struct with the contents and associated metadata describing it + def initialize(@lines : Array(TranscriptLine), @language_code : String, @auto_generated : Bool) + end + + # Generates a protobuf string to fetch the requested transcript from YouTube def self.generate_param(video_id : String, language_code : String, auto_generated : Bool) : String kind = auto_generated ? "asr" : "" @@ -30,48 +43,57 @@ module Invidious::Videos return params end - def self.convert_transcripts_to_vtt(initial_data : Hash(String, JSON::Any), target_language : String) : String - # Convert into array of TranscriptLine - lines = self.parse(initial_data) + # Constructs a Transcripts struct from the initial YouTube response + def self.from_raw(initial_data : Hash(String, JSON::Any), language_code : String, auto_generated : Bool) + body = initial_data.dig("actions", 0, "updateEngagementPanelAction", "content", "transcriptRenderer", + "content", "transcriptSearchPanelRenderer", "body", "transcriptSegmentListRenderer", + "initialSegments").as_a + lines = [] of TranscriptLine + + body.each do |line| + if unpacked_line = line["transcriptSectionHeaderRenderer"]? + line_type = HeadingLine + else + unpacked_line = line["transcriptSegmentRenderer"] + line_type = RegularLine + end + + start_ms = unpacked_line["startMs"].as_s.to_i.millisecond + end_ms = unpacked_line["endMs"].as_s.to_i.millisecond + text = extract_text(unpacked_line["snippet"]) || "" + + lines << line_type.new(start_ms, end_ms, text) + end + + return Transcript.new( + lines: lines, + language_code: language_code, + auto_generated: auto_generated, + ) + end + + # Converts transcript lines to a WebVTT file + # + # This is used within Invidious to replace subtitles + # as to workaround YouTube's rate-limited timedtext endpoint. + def to_vtt settings_field = { "Kind" => "captions", - "Language" => target_language, + "Language" => @language_code, } - # Taken from Invidious::Videos::Captions::Metadata.timedtext_to_vtt() vtt = WebVTT.build(settings_field) do |vtt| - lines.each do |line| + @lines.each do |line| + # Section headers are excluded from the VTT conversion as to + # match the regular captions returned from YouTube as much as possible + next if line.is_a? HeadingLine + vtt.cue(line.start_ms, line.end_ms, line.line) end end return vtt end - - private def self.parse(initial_data : Hash(String, JSON::Any)) - body = initial_data.dig("actions", 0, "updateEngagementPanelAction", "content", "transcriptRenderer", - "content", "transcriptSearchPanelRenderer", "body", "transcriptSegmentListRenderer", - "initialSegments").as_a - - lines = [] of TranscriptLine - body.each do |line| - # Transcript section headers. They are not apart of the captions and as such we can safely skip them. - if line.as_h.has_key?("transcriptSectionHeaderRenderer") - next - end - - line = line["transcriptSegmentRenderer"] - - start_ms = line["startMs"].as_s.to_i.millisecond - end_ms = line["endMs"].as_s.to_i.millisecond - - text = extract_text(line["snippet"]) || "" - - lines << TranscriptLine.new(start_ms, end_ms, text) - end - - return lines - end end end From 5b519123a76879edca3d5fa5cff717b58482e7e5 Mon Sep 17 00:00:00 2001 From: syeopite Date: Tue, 11 Jun 2024 18:46:34 -0700 Subject: [PATCH 17/29] Raise error when transcript does not exist --- src/invidious/videos/transcript.cr | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/invidious/videos/transcript.cr b/src/invidious/videos/transcript.cr index 42f29f46..76fb8610 100644 --- a/src/invidious/videos/transcript.cr +++ b/src/invidious/videos/transcript.cr @@ -45,13 +45,19 @@ module Invidious::Videos # Constructs a Transcripts struct from the initial YouTube response def self.from_raw(initial_data : Hash(String, JSON::Any), language_code : String, auto_generated : Bool) - body = initial_data.dig("actions", 0, "updateEngagementPanelAction", "content", "transcriptRenderer", - "content", "transcriptSearchPanelRenderer", "body", "transcriptSegmentListRenderer", - "initialSegments").as_a + segment_list = initial_data.dig("actions", 0, "updateEngagementPanelAction", "content", "transcriptRenderer", + "content", "transcriptSearchPanelRenderer", "body", "transcriptSegmentListRenderer" + ) + + if !segment_list["initialSegments"]? + raise NotFoundException.new("Requested transcript does not exist") + end + + initial_segments = segment_list["initialSegments"].as_a lines = [] of TranscriptLine - body.each do |line| + initial_segments.each do |line| if unpacked_line = line["transcriptSectionHeaderRenderer"]? line_type = HeadingLine else From f466116cd715120a8acea2c388e306caaf62abb0 Mon Sep 17 00:00:00 2001 From: syeopite Date: Thu, 13 Jun 2024 09:05:47 -0700 Subject: [PATCH 18/29] Extract label for transcript in YouTube response --- src/invidious/videos/transcript.cr | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/src/invidious/videos/transcript.cr b/src/invidious/videos/transcript.cr index 76fb8610..9cd064c5 100644 --- a/src/invidious/videos/transcript.cr +++ b/src/invidious/videos/transcript.cr @@ -8,11 +8,16 @@ module Invidious::Videos alias TranscriptLine = HeadingLine | RegularLine property lines : Array(TranscriptLine) + property language_code : String property auto_generated : Bool + # User friendly label for the current transcript. + # Example: "English (auto-generated)" + property label : String + # Initializes a new Transcript struct with the contents and associated metadata describing it - def initialize(@lines : Array(TranscriptLine), @language_code : String, @auto_generated : Bool) + def initialize(@lines : Array(TranscriptLine), @language_code : String, @auto_generated : Bool, @label : String) end # Generates a protobuf string to fetch the requested transcript from YouTube @@ -45,14 +50,29 @@ module Invidious::Videos # Constructs a Transcripts struct from the initial YouTube response def self.from_raw(initial_data : Hash(String, JSON::Any), language_code : String, auto_generated : Bool) - segment_list = initial_data.dig("actions", 0, "updateEngagementPanelAction", "content", "transcriptRenderer", - "content", "transcriptSearchPanelRenderer", "body", "transcriptSegmentListRenderer" - ) + transcript_panel = initial_data.dig("actions", 0, "updateEngagementPanelAction", "content", "transcriptRenderer", + "content", "transcriptSearchPanelRenderer") + + segment_list = transcript_panel.dig("body", "transcriptSegmentListRenderer") if !segment_list["initialSegments"]? raise NotFoundException.new("Requested transcript does not exist") end + # Extract user-friendly label for the current transcript + + footer_language_menu = transcript_panel.dig?( + "footer", "transcriptFooterRenderer", "languageMenu", "sortFilterSubMenuRenderer", "subMenuItems" + ) + + if footer_language_menu + label = footer_language_menu.as_a.select(&.["selected"].as_bool)[0]["title"].as_s + else + label = language_code + end + + # Extract transcript lines + initial_segments = segment_list["initialSegments"].as_a lines = [] of TranscriptLine @@ -76,6 +96,7 @@ module Invidious::Videos lines: lines, language_code: language_code, auto_generated: auto_generated, + label: label ) end From 45fd4a1968e7c19b1366eb6c05f370efbd2756cd Mon Sep 17 00:00:00 2001 From: syeopite Date: Sun, 16 Jun 2024 13:11:48 -0700 Subject: [PATCH 19/29] Add job to lint code through Ameba in CI --- .github/workflows/ci.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 057e4d61..0db0cb75 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -124,4 +124,26 @@ jobs: - name: Test Docker run: while curl -Isf http://localhost:3000; do sleep 1; done + ameba_lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Install Crystal + uses: crystal-lang/install-crystal@v1.8.0 + with: + crystal: latest + + - name: Cache Shards + uses: actions/cache@v3 + with: + path: ./lib + key: shards-${{ hashFiles('shard.lock') }} + + - name: Install Shards + run: shards install + + - name: Run Ameba linter + run: bin/ameba From a644d76497ad48baf7a5d0230151fdcc4eb33414 Mon Sep 17 00:00:00 2001 From: syeopite Date: Sun, 16 Jun 2024 13:12:15 -0700 Subject: [PATCH 20/29] Update ameba config --- .ameba.yml | 58 +++++++++++------------------------------------------- 1 file changed, 11 insertions(+), 47 deletions(-) diff --git a/.ameba.yml b/.ameba.yml index 96cbc8f0..c7629dcb 100644 --- a/.ameba.yml +++ b/.ameba.yml @@ -20,6 +20,9 @@ Lint/ShadowingOuterLocalVar: Excluded: - src/invidious/helpers/tokens.cr +Lint/NotNil: + Enabled: false + # # Style @@ -31,6 +34,13 @@ Style/RedundantBegin: Style/RedundantReturn: Enabled: false +Style/ParenthesesAroundCondition: + Enabled: false + +# This requires a rewrite of most data structs (and their usage) in Invidious. +Style/QueryBoolMethods: + Enabled: false + # # Metrics @@ -39,50 +49,4 @@ Style/RedundantReturn: # Ignore function complexity (number of if/else & case/when branches) # For some functions that can hardly be simplified for now Metrics/CyclomaticComplexity: - Excluded: - # get_about_info(ucid, locale) => [17/10] - - src/invidious/channels/about.cr - - # fetch_channel_community(ucid, continuation, ...) => [34/10] - - src/invidious/channels/community.cr - - # create_notification_stream(env, topics, connection_channel) => [14/10] - - src/invidious/helpers/helpers.cr:84:5 - - # get_index(plural_form, count) => [25/10] - - src/invidious/helpers/i18next.cr - - # call(context) => [18/10] - - src/invidious/helpers/static_file_handler.cr - - # show(env) => [38/10] - - src/invidious/routes/embed.cr - - # get_video_playback(env) => [45/10] - - src/invidious/routes/video_playback.cr - - # handle(env) => [40/10] - - src/invidious/routes/watch.cr - - # playlist_ajax(env) => [24/10] - - src/invidious/routes/playlists.cr - - # fetch_youtube_comments(id, cursor, ....) => [40/10] - # template_youtube_comments(comments, locale, ...) => [16/10] - # content_to_comment_html(content) => [14/10] - - src/invidious/comments.cr - - # to_json(locale, json) => [21/10] - # extract_video_info(video_id, ...) => [44/10] - # process_video_params(query, preferences) => [20/10] - - src/invidious/videos.cr - - - -#src/invidious/playlists.cr:327:5 -#[C] Metrics/CyclomaticComplexity: Cyclomatic complexity too high [19/10] -# fetch_playlist(plid : String) - -#src/invidious/playlists.cr:436:5 -#[C] Metrics/CyclomaticComplexity: Cyclomatic complexity too high [11/10] -# extract_playlist_videos(initial_data : Hash(String, JSON::Any)) + Enabled: false From e0ed094cc46c6e3e7b37e5e3fbfc8bea9bc267a6 Mon Sep 17 00:00:00 2001 From: syeopite Date: Sun, 16 Jun 2024 13:29:06 -0700 Subject: [PATCH 21/29] Cache ameba binary --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0db0cb75..eb18f639 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -139,7 +139,9 @@ jobs: - name: Cache Shards uses: actions/cache@v3 with: - path: ./lib + path: | + ./lib + ./bin key: shards-${{ hashFiles('shard.lock') }} - name: Install Shards From 6b429575bfa8adcdf2a57a09312c7700237d8a13 Mon Sep 17 00:00:00 2001 From: syeopite Date: Sun, 16 Jun 2024 16:22:01 -0700 Subject: [PATCH 22/29] Update ameba version --- shard.lock | 2 +- shard.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/shard.lock b/shard.lock index efb60a59..397bd8bc 100644 --- a/shard.lock +++ b/shard.lock @@ -2,7 +2,7 @@ version: 2.0 shards: ameba: git: https://github.com/crystal-ameba/ameba.git - version: 1.5.0 + version: 1.6.1 athena-negotiation: git: https://github.com/athena-framework/negotiation.git diff --git a/shard.yml b/shard.yml index be06a7df..367f7c73 100644 --- a/shard.yml +++ b/shard.yml @@ -35,7 +35,7 @@ development_dependencies: version: ~> 0.10.4 ameba: github: crystal-ameba/ameba - version: ~> 1.5.0 + version: ~> 1.6.1 crystal: ">= 1.0.0, < 2.0.0" From 933802b897bb64fec2beebabc696aba4921be68d Mon Sep 17 00:00:00 2001 From: syeopite Date: Mon, 24 Jun 2024 11:34:55 -0700 Subject: [PATCH 23/29] Use "master" label for master container build --- .github/workflows/container-release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/container-release.yml b/.github/workflows/container-release.yml index e44ac200..edb03489 100644 --- a/.github/workflows/container-release.yml +++ b/.github/workflows/container-release.yml @@ -58,7 +58,7 @@ jobs: images: quay.io/invidious/invidious tags: | type=sha,format=short,prefix={{date 'YYYY.MM.DD'}}-,enable=${{ github.ref == format('refs/heads/{0}', 'master') }} - type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'master') }} + type=raw,value=master,enable=${{ github.ref == format('refs/heads/{0}', 'master') }} labels: | quay.expires-after=12w @@ -83,7 +83,7 @@ jobs: suffix=-arm64 tags: | type=sha,format=short,prefix={{date 'YYYY.MM.DD'}}-,enable=${{ github.ref == format('refs/heads/{0}', 'master') }} - type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'master') }} + type=raw,value=master,enable=${{ github.ref == format('refs/heads/{0}', 'master') }} labels: | quay.expires-after=12w From 848ab1e9c866f8e55467697e18a4ef35503cc936 Mon Sep 17 00:00:00 2001 From: syeopite Date: Mon, 24 Jun 2024 11:36:11 -0700 Subject: [PATCH 24/29] Specify which workflow builds from master --- .github/workflows/container-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/container-release.yml b/.github/workflows/container-release.yml index edb03489..55a791b6 100644 --- a/.github/workflows/container-release.yml +++ b/.github/workflows/container-release.yml @@ -1,4 +1,4 @@ -name: Build and release container +name: Build and release container directly from master on: push: From dd38eef41aefd5dd0dc40f21489b9b6cb8269333 Mon Sep 17 00:00:00 2001 From: syeopite Date: Mon, 24 Jun 2024 11:45:00 -0700 Subject: [PATCH 25/29] Add workflow to build container on release --- .github/workflows/release-container.yml | 89 +++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 .github/workflows/release-container.yml diff --git a/.github/workflows/release-container.yml b/.github/workflows/release-container.yml new file mode 100644 index 00000000..9129c699 --- /dev/null +++ b/.github/workflows/release-container.yml @@ -0,0 +1,89 @@ +name: Build and release container + +on: + tags: + - "v*" + +jobs: + release: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Crystal + uses: crystal-lang/install-crystal@v1.8.2 + with: + crystal: 1.12.2 + + - name: Run lint + run: | + if ! crystal tool format --check; then + crystal tool format + git diff + exit 1 + fi + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + with: + platforms: arm64 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to registry + uses: docker/login-action@v3 + with: + registry: quay.io + username: ${{ secrets.QUAY_USERNAME }} + password: ${{ secrets.QUAY_PASSWORD }} + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: quay.io/invidious/invidious + tags: | + type=semvar,pattern={{version}} + type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'master') }} + labels: | + quay.expires-after=12w + + - name: Build and push Docker AMD64 image for Push Event + uses: docker/build-push-action@v5 + with: + context: . + file: docker/Dockerfile + platforms: linux/amd64 + labels: ${{ steps.meta.outputs.labels }} + push: true + tags: ${{ steps.meta.outputs.tags }} + build-args: | + "release=1" + + - name: Docker meta + id: meta-arm64 + uses: docker/metadata-action@v5 + with: + images: quay.io/invidious/invidious + flavor: | + suffix=-arm64 + tags: | + type=semvar,pattern={{version}} + type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'master') }} + labels: | + quay.expires-after=12w + + - name: Build and push Docker ARM64 image for Push Event + uses: docker/build-push-action@v5 + with: + context: . + file: docker/Dockerfile.arm64 + platforms: linux/arm64/v8 + labels: ${{ steps.meta-arm64.outputs.labels }} + push: true + tags: ${{ steps.meta-arm64.outputs.tags }} + build-args: | + "release=1" From 8f5c6a602b78e34e42d1c58ed888e9c8a70ddaa7 Mon Sep 17 00:00:00 2001 From: syeopite Date: Mon, 1 Jul 2024 21:35:08 -0700 Subject: [PATCH 26/29] Rename container workflows --- .../{container-release.yml => build-nightly-container.yml} | 0 .../{release-container.yml => build-stable-container.yml} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{container-release.yml => build-nightly-container.yml} (100%) rename .github/workflows/{release-container.yml => build-stable-container.yml} (100%) diff --git a/.github/workflows/container-release.yml b/.github/workflows/build-nightly-container.yml similarity index 100% rename from .github/workflows/container-release.yml rename to .github/workflows/build-nightly-container.yml diff --git a/.github/workflows/release-container.yml b/.github/workflows/build-stable-container.yml similarity index 100% rename from .github/workflows/release-container.yml rename to .github/workflows/build-stable-container.yml From 64d1f26eceb735ab5c96644b6545fe7fe5c2e677 Mon Sep 17 00:00:00 2001 From: syeopite Date: Mon, 1 Jul 2024 21:39:14 -0700 Subject: [PATCH 27/29] Fix trigger for stable container build --- .github/workflows/build-stable-container.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-stable-container.yml b/.github/workflows/build-stable-container.yml index 9129c699..032fd762 100644 --- a/.github/workflows/build-stable-container.yml +++ b/.github/workflows/build-stable-container.yml @@ -1,8 +1,9 @@ name: Build and release container on: - tags: - - "v*" + push: + tags: + - "v*" jobs: release: From aace30b2b47b715021a2ab661f9dec4b727604b8 Mon Sep 17 00:00:00 2001 From: syeopite Date: Thu, 4 Jul 2024 10:11:36 -0700 Subject: [PATCH 28/29] Bump nightly container build workflow crystal ver --- .github/workflows/build-nightly-container.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-nightly-container.yml b/.github/workflows/build-nightly-container.yml index 55a791b6..bee27600 100644 --- a/.github/workflows/build-nightly-container.yml +++ b/.github/workflows/build-nightly-container.yml @@ -24,9 +24,9 @@ jobs: uses: actions/checkout@v4 - name: Install Crystal - uses: crystal-lang/install-crystal@v1.8.0 + uses: crystal-lang/install-crystal@v1.8.2 with: - crystal: 1.9.2 + crystal: 1.12.2 - name: Run lint run: | From 220cc9bd2ff87872763899f730a8ba62d45db7dd Mon Sep 17 00:00:00 2001 From: syeopite Date: Thu, 4 Jul 2024 10:13:01 -0700 Subject: [PATCH 29/29] Typo Co-authored-by: Samantaz Fox --- .github/workflows/build-stable-container.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-stable-container.yml b/.github/workflows/build-stable-container.yml index 032fd762..b5fbc705 100644 --- a/.github/workflows/build-stable-container.yml +++ b/.github/workflows/build-stable-container.yml @@ -47,7 +47,7 @@ jobs: with: images: quay.io/invidious/invidious tags: | - type=semvar,pattern={{version}} + type=semver,pattern={{version}} type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'master') }} labels: | quay.expires-after=12w @@ -72,7 +72,7 @@ jobs: flavor: | suffix=-arm64 tags: | - type=semvar,pattern={{version}} + type=semver,pattern={{version}} type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'master') }} labels: | quay.expires-after=12w