diff --git a/.github/workflows/build-nightly-container.yml b/.github/workflows/build-nightly-container.yml index e37b9c96..20bbe3d5 100644 --- a/.github/workflows/build-nightly-container.yml +++ b/.github/workflows/build-nightly-container.yml @@ -86,7 +86,7 @@ jobs: # https://github.com/marketplace/actions/docker-manifest-create-action - name: Create and push manifest - uses: int128/docker-manifest-create-action@v2.21.0 + uses: int128/docker-manifest-create-action@v2.22.0 with: push: true tags: quay.io/invidious/invidious:master diff --git a/.github/workflows/build-stable-container.yml b/.github/workflows/build-stable-container.yml index 9058d914..befdf2a1 100644 --- a/.github/workflows/build-stable-container.yml +++ b/.github/workflows/build-stable-container.yml @@ -78,7 +78,7 @@ jobs: # https://github.com/marketplace/actions/docker-manifest-create-action - name: Create and push manifest - uses: int128/docker-manifest-create-action@v2.21.0 + uses: int128/docker-manifest-create-action@v2.22.0 with: push: true tags: quay.io/invidious/invidious:latest diff --git a/assets/css/default.css b/assets/css/default.css index ff07bdb4..726a4c37 100644 --- a/assets/css/default.css +++ b/assets/css/default.css @@ -902,4 +902,11 @@ h1, h2, h3, h4, h5, p, .error-issue-template { padding: 20px; background: rgba(0, 0, 0, 0.12345); +} + +.preference-description { + width: 250px; + padding-left: 10px; + display: inline-block; + vertical-align: top; } \ No newline at end of file diff --git a/assets/js/player.js b/assets/js/player.js index 455b557c..805ae6ef 100644 --- a/assets/js/player.js +++ b/assets/js/player.js @@ -133,7 +133,7 @@ var timeupdate_last_ts = 5; player.on('timeupdate', function () { // Only update once every second let current_ts = Math.floor(player.currentTime()); - if (current_ts > timeupdate_last_ts) timeupdate_last_ts = current_ts; + if (current_ts != timeupdate_last_ts) timeupdate_last_ts = current_ts; else return; // YouTube links diff --git a/config/config.example.yml b/config/config.example.yml index 08005a12..5d76acc1 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -151,6 +151,26 @@ db: ## domain: +## +## List of alternative domains where the invidious instance is being served. +## This needs to be set in order to be able to login and update user preferences +## when using a domain that is not the same as the `domain` configuration, +## like a .`onion` address, `.i2p` address, `.b32.i2p` address, etc. +## +## It will detect the alternative domain trough the `X-Forwarded-Host` header. +## https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/X-Forwarded-Host +## +## Accepted values: a list of fully qualified domain names (FQDN) +## Default: +## +## Example: +## alternative_domains: +## - invidious.example.com +## - inv.example.com +## - videos.example.com +## +alternative_domains: + ## ## Tell Invidious that it is behind a proxy that provides only ## HTTPS, so all links must use the https:// scheme. This diff --git a/docker/Dockerfile b/docker/Dockerfile index 1633393b..fd83cbc5 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -2,7 +2,7 @@ ARG OPENSSL_VERSION='3.6.2' ARG OPENSSL_SHA256='aaf51a1fe064384f811daeaeb4ec4dce7340ec8bd893027eee676af31e83a04f' -FROM crystallang/crystal:1.20.2-alpine AS dependabot-crystal +FROM 84codes/crystal:1.20.2-alpine AS dependabot-crystal # We compile openssl ourselves due to a memory leak in how crystal interacts # with openssl diff --git a/locales/en-US.json b/locales/en-US.json index 5b2ef8d0..d05c0d16 100644 --- a/locales/en-US.json +++ b/locales/en-US.json @@ -124,6 +124,8 @@ "preferences_sort_label": "Sort videos by: ", "preferences_default_playlist": "Default playlist: ", "preferences_default_playlist_none": "No default playlist set", + "preferences_search_privacy_label": "Search privacy: ", + "preferences_search_privacy_description": "Enabling this preference will prevent your search queries from being saved in your browser history.", "published": "published", "published - reverse": "published - reverse", "alphabetically": "alphabetically", diff --git a/spec/invidious/videos/regular_videos_extract_spec.cr b/spec/invidious/videos/regular_videos_extract_spec.cr index b82a08ee..00b85bd0 100644 --- a/spec/invidious/videos/regular_videos_extract_spec.cr +++ b/spec/invidious/videos/regular_videos_extract_spec.cr @@ -7,7 +7,7 @@ Spectator.describe "parse_video_info" do _next = load_mock("video/regular_mrbeast.next") raw_data = _player.merge!(_next) - info = parse_video_info("2isYuQZMbdU", raw_data) + info = Invidious::Videos::Parser.parse_video_info("2isYuQZMbdU", raw_data) # Some basic verifications expect(typeof(info)).to eq(Hash(String, JSON::Any)) @@ -88,7 +88,7 @@ Spectator.describe "parse_video_info" do _next = load_mock("video/regular_no-description.next") raw_data = _player.merge!(_next) - info = parse_video_info("iuevw6218F0", raw_data) + info = Invidious::Videos::Parser.parse_video_info("iuevw6218F0", raw_data) # Some basic verifications expect(typeof(info)).to eq(Hash(String, JSON::Any)) diff --git a/spec/invidious/videos/scheduled_live_extract_spec.cr b/spec/invidious/videos/scheduled_live_extract_spec.cr index 6bb03e42..95c90813 100644 --- a/spec/invidious/videos/scheduled_live_extract_spec.cr +++ b/spec/invidious/videos/scheduled_live_extract_spec.cr @@ -7,7 +7,7 @@ Spectator.describe "parse_video_info" do _next = load_mock("video/scheduled_live_PBD-Podcast.next") raw_data = _player.merge!(_next) - info = parse_video_info("N-yVic7BbY0", raw_data) + info = Invidious::Videos::Parser.parse_video_info("N-yVic7BbY0", raw_data) # Some basic verifications expect(typeof(info)).to eq(Hash(String, JSON::Any)) diff --git a/src/invidious/channels/about.cr b/src/invidious/channels/about.cr index 537aa034..bb55147b 100644 --- a/src/invidious/channels/about.cr +++ b/src/invidious/channels/about.cr @@ -83,11 +83,16 @@ def get_about_info(ucid, locale) : AboutChannel author = initdata["metadata"]["channelMetadataRenderer"]["title"].as_s author_url = initdata["metadata"]["channelMetadataRenderer"]["channelUrl"].as_s author_thumbnail = initdata["metadata"]["channelMetadataRenderer"]["avatar"]["thumbnails"][0]["url"].as_s - author_verified = has_verified_badge?(initdata.dig?("header", "c4TabbedHeaderRenderer", "badges")) - + author_badge = initdata.dig?("header", "pageHeaderRenderer", "content", "pageHeaderViewModel", "title", "dynamicTextViewModel", "text", "attachmentRuns", 0, "element", "type", "imageType", "image", "sources", 0, "clientResource", "imageName") + .try &.as_s + # CHECK_CIRCLE_FILLED is used for normal channels and AUDIO_BADGE if used For + # music/artist channels + # TODO: Maybe separate verified author from verified artist? + author_verified = author_badge.try { |badge| badge == "CHECK_CIRCLE_FILLED" || badge == "AUDIO_BADGE" } || false ucid = initdata["metadata"]["channelMetadataRenderer"]["externalId"].as_s # Raises a KeyError on failure. + # TODO: Check if `c4TabbedHeaderRenderer` still exists on some channels. banners = initdata["header"]["c4TabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]? banners ||= initdata.dig?("header", "pageHeaderRenderer", "content", "pageHeaderViewModel", "banner", "imageBannerViewModel", "image", "sources") banner = banners.try &.[-1]?.try &.["url"].as_s? diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 2ab0504d..69f1ff00 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -69,6 +69,7 @@ struct ConfigPreferences property save_player_pos : Bool = false @[YAML::Field(ignore: true)] property default_playlist : String? = nil + property search_privacy : Bool = false def to_tuple {% begin %} @@ -135,6 +136,8 @@ class Config property hmac_key : String = "" # Domain to be used for links to resources on the site where an absolute URL is required property domain : String? + # Additional domain list that is going to be used for cookie domain validation + property alternative_domains : Array(String) = [] of String # Subscribe to channels using PubSubHubbub (requires domain, hmac_key) property use_pubsub_feeds : Bool | Int32 = false property popular_enabled : Bool = true diff --git a/src/invidious/helpers/errors.cr b/src/invidious/helpers/errors.cr index 97e28022..cee6eb2c 100644 --- a/src/invidious/helpers/errors.cr +++ b/src/invidious/helpers/errors.cr @@ -201,7 +201,7 @@ def error_redirect_helper(env : HTTP::Server::Context) #{switch_instance}
  • - #{go_to_youtube} + #{go_to_youtube}
    • END_HTML diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr index 5f98321b..0a5d5a4e 100644 --- a/src/invidious/routes/api/v1/videos.cr +++ b/src/invidious/routes/api/v1/videos.cr @@ -410,7 +410,7 @@ module Invidious::Routes::API::V1::Videos clip_title = nil if params = response.dig?("endpoint", "watchEndpoint", "params").try &.as_s - start_time, end_time, clip_title = parse_clip_parameters(params) + start_time, end_time, clip_title = Invidious::Videos::Clip.parse_clip_parameters(params) end begin diff --git a/src/invidious/routes/before_all.cr b/src/invidious/routes/before_all.cr index 45b8c91c..b75574b6 100644 --- a/src/invidious/routes/before_all.cr +++ b/src/invidious/routes/before_all.cr @@ -32,6 +32,8 @@ module Invidious::Routes::BeforeAll env.response.headers["X-XSS-Protection"] = "1; mode=block" env.response.headers["X-Content-Type-Options"] = "nosniff" + env.set "header_x-forwarded-host", env.request.headers["X-Forwarded-Host"]? + # Only allow the pages at /embed/* to be embedded if env.request.resource.starts_with?("/embed") frame_ancestors = "'self' file: http: https:" diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr index a9b476bc..0477802a 100644 --- a/src/invidious/routes/channels.cr +++ b/src/invidious/routes/channels.cr @@ -351,7 +351,7 @@ module Invidious::Routes::Channels invidious_url_params.delete_all("user") begin - resolved_url = YoutubeAPI.resolve_url("https://youtube.com#{env.request.path}#{yt_url_params.size > 0 ? "?#{yt_url_params}" : ""}") + resolved_url = YoutubeAPI.resolve_url("https://www.youtube.com#{env.request.path}#{yt_url_params.size > 0 ? "?#{yt_url_params}" : ""}") ucid = resolved_url["endpoint"]["browseEndpoint"]["browseId"] rescue ex : InfoException | KeyError return error_template(404, I18n.translate(locale, "This channel does not exist.")) diff --git a/src/invidious/routes/embed.cr b/src/invidious/routes/embed.cr index ccd4136f..41dc517c 100644 --- a/src/invidious/routes/embed.cr +++ b/src/invidious/routes/embed.cr @@ -122,7 +122,7 @@ module Invidious::Routes::Embed else nil # Continue end - params = process_video_params(env.params.query, preferences) + params = Invidious::Videos.process_video_params(env.params.query, preferences) user = env.get?("user").try &.as(User) if user diff --git a/src/invidious/routes/errors.cr b/src/invidious/routes/errors.cr index 1e9ab44e..e8e7e287 100644 --- a/src/invidious/routes/errors.cr +++ b/src/invidious/routes/errors.cr @@ -8,7 +8,7 @@ module Invidious::Routes::ErrorRoutes if md = env.request.path.match(/^\/(?([a-zA-Z0-9_-]{11})|(\w+))$/) item = md["id"] - # Check if item is branding URL e.g. https://youtube.com/gaming + # Check if item is branding URL e.g. https://www.youtube.com/gaming response = YT_POOL.client &.get("/#{item}") if response.status_code == 301 diff --git a/src/invidious/routes/feeds.cr b/src/invidious/routes/feeds.cr index a8247d78..b214c991 100644 --- a/src/invidious/routes/feeds.cr +++ b/src/invidious/routes/feeds.cr @@ -320,7 +320,7 @@ module Invidious::Routes::Feeds case attribute.name when "url", "href" request_target = URI.parse(node[attribute.name]).request_target - query_string_opt = request_target.starts_with?("/watch?v=") ? "&#{params}" : "" + query_string_opt = request_target.starts_with?("/watch?v=") ? ("&#{params}" if !params.empty?) : "" node[attribute.name] = "#{HOST_URL}#{request_target}#{query_string_opt}" else nil # Skip end diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr index c06955c0..d76bcc58 100644 --- a/src/invidious/routes/images.cr +++ b/src/invidious/routes/images.cr @@ -51,7 +51,7 @@ module Invidious::Routes::Images end # ??? maybe also for storyboards? - def self.s_p_image(env) + def self.s_p_image(env, authority = "i9") id = env.params.url["id"] name = env.params.url["name"] url = env.request.resource @@ -65,13 +65,23 @@ module Invidious::Routes::Images end begin - get_ytimg_pool("i9").client &.get(url, headers) do |resp| + get_ytimg_pool(authority).client &.get(url, headers) do |resp| return self.proxy_image(env, resp) end rescue ex end end + # Both pl_c and tvfilm_banner use the same logic used in s_p_image(env) + # just with a different authority ("i"). + def self.pl_c_image(env) + self.s_p_image(env, "i") + end + + def self.tvfilm_banner_image(env) + self.s_p_image(env, "i") + end + def self.yts_image(env) headers = HTTP::Headers.new REQUEST_HEADERS_WHITELIST.each do |header| diff --git a/src/invidious/routes/login.cr b/src/invidious/routes/login.cr index 5fb800b5..aae181d8 100644 --- a/src/invidious/routes/login.cr +++ b/src/invidious/routes/login.cr @@ -35,6 +35,7 @@ module Invidious::Routes::Login def self.login(env) locale = env.get("preferences").as(Preferences).locale + host = env.get("header_x-forwarded-host") referer = get_referer(env, "/feed/subscriptions") @@ -82,7 +83,11 @@ module Invidious::Routes::Login sid = Base64.urlsafe_encode(Random::Secure.random_bytes(32)) Invidious::Database::SessionIDs.insert(sid, email) - env.response.cookies["SID"] = Invidious::User::Cookies.sid(CONFIG.domain, sid) + if alt = CONFIG.alternative_domains.index(host) + env.response.cookies["SID"] = Invidious::User::Cookies.sid(CONFIG.alternative_domains[alt], sid) + else + env.response.cookies["SID"] = Invidious::User::Cookies.sid(CONFIG.domain, sid) + end else return error_template(401, "Wrong username or password") end @@ -148,7 +153,11 @@ module Invidious::Routes::Login view_name = "subscriptions_#{sha256(user.email)}" PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}") - env.response.cookies["SID"] = Invidious::User::Cookies.sid(CONFIG.domain, sid) + if alt = CONFIG.alternative_domains.index(host) + env.response.cookies["SID"] = Invidious::User::Cookies.sid(CONFIG.alternative_domains[alt], sid) + else + env.response.cookies["SID"] = Invidious::User::Cookies.sid(CONFIG.domain, sid) + end if env.request.cookies["PREFS"]? user.preferences = env.get("preferences").as(Preferences) diff --git a/src/invidious/routes/preferences.cr b/src/invidious/routes/preferences.cr index d9fad1b1..77ea02e9 100644 --- a/src/invidious/routes/preferences.cr +++ b/src/invidious/routes/preferences.cr @@ -145,6 +145,10 @@ module Invidious::Routes::PreferencesRoute default_playlist = env.params.body["default_playlist"]?.try &.as(String) + search_privacy = env.params.body["search_privacy"]?.try &.as(String) + search_privacy ||= "off" + search_privacy = search_privacy == "on" + # Convert to JSON and back again to take advantage of converters used for compatibility preferences = Preferences.from_json({ annotations: annotations, @@ -182,6 +186,7 @@ module Invidious::Routes::PreferencesRoute show_nick: show_nick, save_player_pos: save_player_pos, default_playlist: default_playlist, + search_privacy: search_privacy, }.to_json) if user = env.get? "user" @@ -226,7 +231,12 @@ module Invidious::Routes::PreferencesRoute File.write("config/config.yml", CONFIG.to_yaml) end else - env.response.cookies["PREFS"] = Invidious::User::Cookies.prefs(CONFIG.domain, preferences) + host = env.get("header_x-forwarded-host") + if alt = CONFIG.alternative_domains.index(host) + env.response.cookies["PREFS"] = Invidious::User::Cookies.prefs(CONFIG.alternative_domains[alt], preferences) + else + env.response.cookies["PREFS"] = Invidious::User::Cookies.prefs(CONFIG.domain, preferences) + end end env.redirect referer @@ -261,7 +271,12 @@ module Invidious::Routes::PreferencesRoute preferences.dark_mode = "dark" end - env.response.cookies["PREFS"] = Invidious::User::Cookies.prefs(CONFIG.domain, preferences) + host = env.get("header_x-forwarded-host") + if alt = CONFIG.alternative_domains.index(host) + env.response.cookies["PREFS"] = Invidious::User::Cookies.prefs(CONFIG.alternative_domains[alt], preferences) + else + env.response.cookies["PREFS"] = Invidious::User::Cookies.prefs(CONFIG.domain, preferences) + end end if redirect diff --git a/src/invidious/routes/search.cr b/src/invidious/routes/search.cr index 11e6f171..968a5554 100644 --- a/src/invidious/routes/search.cr +++ b/src/invidious/routes/search.cr @@ -40,9 +40,16 @@ module Invidious::Routes::Search preferences = env.get("preferences").as(Preferences) locale = preferences.locale - region = env.params.query["region"]? || preferences.region + uri_params = URI::Params.new + if env.request.method == "GET" + uri_params = env.params.query + else + uri_params = env.params.body + end - query = Invidious::Search::Query.new(env.params.query, :regular, region) + region = uri_params["region"]? || preferences.region + + query = Invidious::Search::Query.new(uri_params, :regular, region) if query.empty? # Display the full page search box implemented in #1977 diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index b829b0f5..7a68a145 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -47,7 +47,7 @@ module Invidious::Routes::Watch end subscriptions ||= [] of String - params = process_video_params(env.params.query, preferences) + params = Invidious::Videos.process_video_params(env.params.query, preferences) env.params.query.delete_all("listen") begin @@ -129,17 +129,20 @@ module Invidious::Routes::Watch video_streams = video.video_streams audio_streams = video.audio_streams - # Older videos may not have audio sources available. - # We redirect here so they're not unplayable - if audio_streams.empty? && !video.live_now - if params.quality == "dash" - env.params.query.delete_all("quality") - env.params.query["quality"] = "medium" - return env.redirect "/watch?#{env.params.query}" - elsif params.listen - env.params.query.delete_all("listen") - env.params.query["listen"] = "0" - return env.redirect "/watch?#{env.params.query}" + # Videos that are a premiere do not have audio streams. + if video.premiere_timestamp.nil? + # Older videos may not have audio sources available. + # We redirect here so they're not unplayable + if audio_streams.empty? && !video.live_now + if params.quality == "dash" + env.params.query.delete_all("quality") + env.params.query["quality"] = "medium" + return env.redirect "/watch?#{env.params.query}" + elsif params.listen + env.params.query.delete_all("listen") + env.params.query["listen"] = "0" + return env.redirect "/watch?#{env.params.query}" + end end end @@ -273,7 +276,7 @@ module Invidious::Routes::Watch if video_id = response.dig?("endpoint", "watchEndpoint", "videoId") if params = response.dig?("endpoint", "watchEndpoint", "params").try &.as_s - start_time, end_time, _ = parse_clip_parameters(params) + start_time, end_time, _ = Invidious::Videos::Clip.parse_clip_parameters(params) env.params.query["start"] = start_time.to_s if start_time != nil env.params.query["end"] = end_time.to_s if end_time != nil end diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr index 68cf32de..43c83d6e 100644 --- a/src/invidious/routing.cr +++ b/src/invidious/routing.cr @@ -186,6 +186,7 @@ module Invidious::Routing get "/opensearch.xml", Routes::Search, :opensearch get "/results", Routes::Search, :results get "/search", Routes::Search, :search + post "/search", Routes::Search, :search get "/hashtag/:hashtag", Routes::Search, :hashtag end @@ -223,6 +224,8 @@ module Invidious::Routing get "/s_p/:id/:name", Routes::Images, :s_p_image get "/yts/img/:name", Routes::Images, :yts_image get "/vi/:id/:name", Routes::Images, :thumbnails + get "/pl_c/:id/:name", Routes::Images, :pl_c_image + get "/tvfilm_banner/:id/:name", Routes::Images, :tvfilm_banner_image end def register_companion_routes diff --git a/src/invidious/user/cookies.cr b/src/invidious/user/cookies.cr index 654efc15..ed7665fd 100644 --- a/src/invidious/user/cookies.cr +++ b/src/invidious/user/cookies.cr @@ -6,17 +6,24 @@ struct Invidious::User # Note: we use ternary operator because the two variables # used in here are not booleans. - SECURE = (Kemal.config.ssl || CONFIG.https_only) ? true : false + @@secure = (Kemal.config.ssl || CONFIG.https_only) ? true : false # Session ID (SID) cookie # Parameter "domain" comes from the global config def sid(domain : String?, sid) : HTTP::Cookie + # Not secure if it's being accessed from I2P + # Browsers expect the domain to include https. On I2P there is no HTTPS + # Tor browser works fine with secure being true + if (domain.try &.split(".").last == "i2p") && @@secure + @@secure = false + end + return HTTP::Cookie.new( name: "SID", domain: domain, value: sid, expires: Time.utc + 2.years, - secure: SECURE, + secure: @@secure, http_only: true, samesite: HTTP::Cookie::SameSite::Lax ) @@ -25,12 +32,19 @@ struct Invidious::User # Preferences (PREFS) cookie # Parameter "domain" comes from the global config def prefs(domain : String?, preferences : Preferences) : HTTP::Cookie + # Not secure if it's being accessed from I2P + # Browsers expect the domain to include https. On I2P there is no HTTPS + # Tor browser works fine with secure being true + if (domain.try &.split(".").last == "i2p") && @@secure + @@secure = false + end + return HTTP::Cookie.new( name: "PREFS", domain: domain, value: URI.encode_www_form(preferences.to_json), expires: Time.utc + 2.years, - secure: SECURE, + secure: @@secure, http_only: false, samesite: HTTP::Cookie::SameSite::Lax ) diff --git a/src/invidious/user/preferences.cr b/src/invidious/user/preferences.cr index df195dd6..ad4a9bc7 100644 --- a/src/invidious/user/preferences.cr +++ b/src/invidious/user/preferences.cr @@ -57,6 +57,7 @@ struct Preferences property volume : Int32 = CONFIG.default_user_preferences.volume property save_player_pos : Bool = CONFIG.default_user_preferences.save_player_pos property default_playlist : String? = nil + property search_privacy : Bool = CONFIG.default_user_preferences.search_privacy module BoolToString def self.to_json(value : String, json : JSON::Builder) diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 0446922f..99713d18 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -81,9 +81,10 @@ struct Video end def premiere_timestamp : Time? - info - .dig?("microformat", "playerMicroformatRenderer", "liveBroadcastDetails", "startTimestamp") - .try { |t| Time.parse_rfc3339(t.as_s) } + if self.video_type == VideoType::Scheduled + return info["published"]? + .try { |t| Time.parse_rfc3339(t.as_s) } + end end def related_videos @@ -324,7 +325,7 @@ rescue DB::Error end def fetch_video(id, region) - info = extract_video_info(video_id: id) + info = Invidious::Videos::Parser.extract_video_info(video_id: id) if info.nil? raise InfoException.new("Invidious companion is not available. \ diff --git a/src/invidious/videos/clip.cr b/src/invidious/videos/clip.cr index 29c57182..080d2e92 100644 --- a/src/invidious/videos/clip.cr +++ b/src/invidious/videos/clip.cr @@ -1,22 +1,26 @@ require "json" -# returns start_time, end_time and clip_title -def parse_clip_parameters(params) : {Float64?, Float64?, String?} - decoded_protobuf = params.try { |i| URI.decode_www_form(i) } - .try { |i| Base64.decode(i) } - .try { |i| IO::Memory.new(i) } - .try { |i| Protodec::Any.parse(i) } +module Invidious::Videos::Clip + extend self - start_time = decoded_protobuf - .try(&.["50:0:embedded"]["2:1:varint"].as_i64) - .try { |i| i/1000 } + # returns start_time, end_time and clip_title + def parse_clip_parameters(params) : {Float64?, Float64?, String?} + decoded_protobuf = params.try { |i| URI.decode_www_form(i) } + .try { |i| Base64.decode(i) } + .try { |i| IO::Memory.new(i) } + .try { |i| Protodec::Any.parse(i) } - end_time = decoded_protobuf - .try(&.["50:0:embedded"]["3:2:varint"].as_i64) - .try { |i| i/1000 } + start_time = decoded_protobuf + .try(&.["50:0:embedded"]["2:1:varint"].as_i64) + .try { |i| i/1000 } - clip_title = decoded_protobuf - .try(&.["50:0:embedded"]["4:3:string"].as_s) + end_time = decoded_protobuf + .try(&.["50:0:embedded"]["3:2:varint"].as_i64) + .try { |i| i/1000 } - return start_time, end_time, clip_title + clip_title = decoded_protobuf + .try(&.["50:0:embedded"]["4:3:string"].as_s) + + return start_time, end_time, clip_title + end end diff --git a/src/invidious/videos/description.cr b/src/invidious/videos/description.cr index 1371bebb..18b4122e 100644 --- a/src/invidious/videos/description.cr +++ b/src/invidious/videos/description.cr @@ -21,8 +21,6 @@ private def copy_string(str : String::Builder, iter : Iterator, count : Int) : I str << cp.chr end - # A codepoint from the SMP counts twice - copied += 1 if cp > 0xFFFF copied += 1 end @@ -44,10 +42,6 @@ def parse_description(desc, video_id : String) : String? end end - # Not everything is stored in UTF-8 on youtube's side. The SMP codepoints - # (0x10000 and above) are encoded as UTF-16 surrogate pairs, which are - # automatically decoded by the JSON parser. It means that we need to count - # copied byte in a special manner, preventing the use of regular string copy. iter = content.each_codepoint index = 0 diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index d9107d5a..914c5963 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -1,465 +1,469 @@ require "json" -# Use to parse both "compactVideoRenderer" and "endScreenVideoRenderer". -# The former is preferred as it has more videos in it. The second has -# the same 11 first entries as the compact rendered. -# -# TODO: "compactRadioRenderer" (Mix) and -# TODO: Use a proper struct/class instead of a hacky JSON object -def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)? - return nil if !related["videoId"]? +module Invidious::Videos::Parser + extend self - # The compact renderer has video length in seconds, where the end - # screen rendered has a full text version ("42:40") - length = related["lengthInSeconds"]?.try &.as_i.to_s - length ||= related.dig?("lengthText", "simpleText").try do |box| - decode_length_seconds(box.as_s).to_s + # Use to parse both "compactVideoRenderer" and "endScreenVideoRenderer". + # The former is preferred as it has more videos in it. The second has + # the same 11 first entries as the compact rendered. + # + # TODO: "compactRadioRenderer" (Mix) and + # TODO: Use a proper struct/class instead of a hacky JSON object + def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)? + return nil if !related["videoId"]? + + # The compact renderer has video length in seconds, where the end + # screen rendered has a full text version ("42:40") + length = related["lengthInSeconds"]?.try &.as_i.to_s + length ||= related.dig?("lengthText", "simpleText").try do |box| + decode_length_seconds(box.as_s).to_s + end + + # Both have "short", so the "long" option shouldn't be required + channel_info = (related["shortBylineText"]? || related["longBylineText"]?) + .try &.dig?("runs", 0) + + author = channel_info.try &.dig?("text") + author_verified = has_verified_badge?(related["ownerBadges"]?).to_s + + ucid = channel_info.try { |ci| HelperExtractors.get_browse_id(ci) } + + short_view_count = related.try do |r| + HelperExtractors.get_short_view_count(r).to_s + end + + LOGGER.trace("parse_related_video: Found \"watchNextEndScreenRenderer\" container") + + if published_time_text = related["publishedTimeText"]? + decoded_time = decode_date(published_time_text["simpleText"].to_s) + published = decoded_time.to_rfc3339.to_s + else + published = nil + end + + # TODO: when refactoring video types, make a struct for related videos + # or reuse an existing type, if that fits. + return { + "id" => related["videoId"], + "title" => related["title"]["simpleText"], + "author" => author || JSON::Any.new(""), + "ucid" => JSON::Any.new(ucid || ""), + "length_seconds" => JSON::Any.new(length || "0"), + "short_view_count" => JSON::Any.new(short_view_count || "0"), + "author_verified" => JSON::Any.new(author_verified), + "published" => JSON::Any.new(published || ""), + } end - # Both have "short", so the "long" option shouldn't be required - channel_info = (related["shortBylineText"]? || related["longBylineText"]?) - .try &.dig?("runs", 0) + def extract_video_info(video_id : String) + # Fetch data from the player endpoint + player_response = YoutubeAPI.player(video_id: video_id) - author = channel_info.try &.dig?("text") - author_verified = has_verified_badge?(related["ownerBadges"]?).to_s + if player_response.nil? + return nil + end - ucid = channel_info.try { |ci| HelperExtractors.get_browse_id(ci) } + playability_status = player_response.dig?("playabilityStatus", "status").try &.as_s - short_view_count = related.try do |r| - HelperExtractors.get_short_view_count(r).to_s - end + if playability_status != "OK" + subreason = player_response.dig?("playabilityStatus", "errorScreen", "playerErrorMessageRenderer", "subreason") + reason = subreason.try &.[]?("simpleText").try &.as_s + reason ||= subreason.try &.[]("runs").as_a.map(&.[]("text")).join("") + reason ||= player_response.dig("playabilityStatus", "reason").as_s - LOGGER.trace("parse_related_video: Found \"watchNextEndScreenRenderer\" container") + # Stop here if video is not a scheduled livestream or + # for LOGIN_REQUIRED when videoDetails element is not found because retrying won't help + if !{"LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED"}.any?(playability_status) || + playability_status == "LOGIN_REQUIRED" && !player_response.dig?("videoDetails") + return { + "version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64), + "reason" => JSON::Any.new(reason), + } + end + elsif video_id != player_response.dig?("videoDetails", "videoId") + # YouTube may return a different video player response than expected. + # See: https://github.com/TeamNewPipe/NewPipe/issues/8713 + # Line to be reverted if one day we solve the video not available issue. - if published_time_text = related["publishedTimeText"]? - decoded_time = decode_date(published_time_text["simpleText"].to_s) - published = decoded_time.to_rfc3339.to_s - else - published = nil - end + # Although technically not a call to /videoplayback the fact that YouTube is returning the + # wrong video means that we should count it as a failure. + Helpers.get_playback_statistic["totalRequests"] += 1 - # TODO: when refactoring video types, make a struct for related videos - # or reuse an existing type, if that fits. - return { - "id" => related["videoId"], - "title" => related["title"]["simpleText"], - "author" => author || JSON::Any.new(""), - "ucid" => JSON::Any.new(ucid || ""), - "length_seconds" => JSON::Any.new(length || "0"), - "short_view_count" => JSON::Any.new(short_view_count || "0"), - "author_verified" => JSON::Any.new(author_verified), - "published" => JSON::Any.new(published || ""), - } -end - -def extract_video_info(video_id : String) - # Fetch data from the player endpoint - player_response = YoutubeAPI.player(video_id: video_id) - - if player_response.nil? - return nil - end - - playability_status = player_response.dig?("playabilityStatus", "status").try &.as_s - - if playability_status != "OK" - subreason = player_response.dig?("playabilityStatus", "errorScreen", "playerErrorMessageRenderer", "subreason") - reason = subreason.try &.[]?("simpleText").try &.as_s - reason ||= subreason.try &.[]("runs").as_a.map(&.[]("text")).join("") - reason ||= player_response.dig("playabilityStatus", "reason").as_s - - # Stop here if video is not a scheduled livestream or - # for LOGIN_REQUIRED when videoDetails element is not found because retrying won't help - if !{"LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED"}.any?(playability_status) || - playability_status == "LOGIN_REQUIRED" && !player_response.dig?("videoDetails") return { "version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64), - "reason" => JSON::Any.new(reason), + "reason" => JSON::Any.new("Can't load the video on this Invidious instance. YouTube is currently trying to block Invidious instances. Click here for more info about the issue."), } + else + reason = nil end - elsif video_id != player_response.dig?("videoDetails", "videoId") - # YouTube may return a different video player response than expected. - # See: https://github.com/TeamNewPipe/NewPipe/issues/8713 - # Line to be reverted if one day we solve the video not available issue. - # Although technically not a call to /videoplayback the fact that YouTube is returning the - # wrong video means that we should count it as a failure. - Helpers.get_playback_statistic["totalRequests"] += 1 + # Don't fetch the next endpoint if the video is unavailable. + if {"OK", "LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED"}.any?(playability_status) + next_response = YoutubeAPI.next({"videoId": video_id, "params": ""}) + # Remove the microformat returned by the /next endpoint on some videos + # to prevent player_response microformat from being overwritten. + next_response.delete("microformat") + player_response = player_response.merge(next_response) + end - return { - "version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64), - "reason" => JSON::Any.new("Can't load the video on this Invidious instance. YouTube is currently trying to block Invidious instances. Click here for more info about the issue."), - } - else - reason = nil - end + params = self.parse_video_info(video_id, player_response) + params["reason"] = JSON::Any.new(reason) if reason - # Don't fetch the next endpoint if the video is unavailable. - if {"OK", "LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED"}.any?(playability_status) - next_response = YoutubeAPI.next({"videoId": video_id, "params": ""}) - # Remove the microformat returned by the /next endpoint on some videos - # to prevent player_response microformat from being overwritten. - next_response.delete("microformat") - player_response = player_response.merge(next_response) - end + {"captions", "playabilityStatus", "playerConfig", "storyboards"}.each do |f| + params[f] = player_response[f] if player_response[f]? + end - params = parse_video_info(video_id, player_response) - params["reason"] = JSON::Any.new(reason) if reason - - {"captions", "playabilityStatus", "playerConfig", "storyboards"}.each do |f| - params[f] = player_response[f] if player_response[f]? - end - - # Convert URLs, if those are present - if streaming_data = player_response["streamingData"]? - %w[formats adaptiveFormats].each do |key| - streaming_data.as_h[key]?.try &.as_a.each do |format| - format = format.as_h - if format["url"]?.nil? - format["url"] = format["signatureCipher"] + # Convert URLs, if those are present + if streaming_data = player_response["streamingData"]? + %w[formats adaptiveFormats].each do |key| + streaming_data.as_h[key]?.try &.as_a.each do |format| + format = format.as_h + if format["url"]?.nil? + format["url"] = format["signatureCipher"] + end + format["url"] = JSON::Any.new(convert_url(format)) end - format["url"] = JSON::Any.new(convert_url(format)) end + + params["streamingData"] = streaming_data end - params["streamingData"] = streaming_data + # Data structure version, for cache control + params["version"] = JSON::Any.new(Video::SCHEMA_VERSION.to_i64) + + return params end - # Data structure version, for cache control - params["version"] = JSON::Any.new(Video::SCHEMA_VERSION.to_i64) + def try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConfig) : Hash(String, JSON::Any)? + LOGGER.debug("try_fetch_streaming_data: [#{id}] Using #{client_config.client_type} client.") + response = YoutubeAPI.player(video_id: id) - return params -end + playability_status = response["playabilityStatus"]["status"] + LOGGER.debug("try_fetch_streaming_data: [#{id}] Got playabilityStatus == #{playability_status}.") -def try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConfig) : Hash(String, JSON::Any)? - LOGGER.debug("try_fetch_streaming_data: [#{id}] Using #{client_config.client_type} client.") - response = YoutubeAPI.player(video_id: id) - - playability_status = response["playabilityStatus"]["status"] - LOGGER.debug("try_fetch_streaming_data: [#{id}] Got playabilityStatus == #{playability_status}.") - - if id != response.dig?("videoDetails", "videoId") - # YouTube may return a different video player response than expected. - # See: https://github.com/TeamNewPipe/NewPipe/issues/8713 - raise InfoException.new( - "The video returned by YouTube isn't the requested one. (#{client_config.client_type} client)" - ) - elsif playability_status == "OK" - return response - else - return nil - end -end - -def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any)) : Hash(String, JSON::Any) - # Top level elements - - main_results = player_response.dig?("contents", "twoColumnWatchNextResults") - - raise BrokenTubeException.new("twoColumnWatchNextResults") if !main_results - - # Primary results are not available on Music videos - # See: https://github.com/iv-org/invidious/pull/3238#issuecomment-1207193725 - if primary_results = main_results.dig?("results", "results", "contents") - video_primary_renderer = primary_results - .as_a.find(&.["videoPrimaryInfoRenderer"]?) - .try &.["videoPrimaryInfoRenderer"] - - video_secondary_renderer = primary_results - .as_a.find(&.["videoSecondaryInfoRenderer"]?) - .try &.["videoSecondaryInfoRenderer"] - - raise BrokenTubeException.new("videoPrimaryInfoRenderer") if !video_primary_renderer - raise BrokenTubeException.new("videoSecondaryInfoRenderer") if !video_secondary_renderer - end - - video_details = player_response.dig?("videoDetails") - if !(microformat = player_response.dig?("microformat", "playerMicroformatRenderer")) - microformat = {} of String => JSON::Any - end - - raise BrokenTubeException.new("videoDetails") if !video_details - - # Basic video infos - - title = video_details["title"]?.try &.as_s - - # We have to try to extract viewCount from videoPrimaryInfoRenderer first, - # then from videoDetails, as the latter is "0" for livestreams (we want - # to get the amount of viewers watching). - views_txt = extract_text( - video_primary_renderer - .try &.dig?("viewCount", "videoViewCountRenderer", "viewCount") - ) - views_txt ||= video_details["viewCount"]?.try &.as_s || "" - views = views_txt.gsub(/\D/, "").to_i64? - - length_txt = (microformat["lengthSeconds"]? || video_details["lengthSeconds"]) - .try &.as_s.to_i64 - - published = microformat["publishDate"]? - .try { |t| Time.parse(t.as_s, "%Y-%m-%d", Time::Location::UTC) } || Time.utc - - premiere_timestamp = microformat.dig?("liveBroadcastDetails", "startTimestamp") - .try { |t| Time.parse_rfc3339(t.as_s) } - - premiere_timestamp ||= player_response.dig?( - "playabilityStatus", "liveStreamability", - "liveStreamabilityRenderer", "offlineSlate", - "liveStreamOfflineSlateRenderer", "scheduledStartTime" - ) - .try &.as_s.to_i64 - .try { |t| Time.unix(t) } - - live_now = microformat.dig?("liveBroadcastDetails", "isLiveNow") - .try &.as_bool - live_now ||= video_details.dig?("isLive").try &.as_bool || false - - post_live_dvr = video_details.dig?("isPostLiveDvr") - .try &.as_bool || false - - # Extra video infos - - allowed_regions = microformat["availableCountries"]? - .try &.as_a.map &.as_s || [] of String - - allow_ratings = video_details["allowRatings"]?.try &.as_bool - family_friendly = microformat["isFamilySafe"]?.try &.as_bool - is_listed = video_details["isCrawlable"]?.try &.as_bool - is_upcoming = video_details["isUpcoming"]?.try &.as_bool - - keywords = video_details["keywords"]? - .try &.as_a.map &.as_s || [] of String - - # Related videos - - LOGGER.debug("extract_video_info: parsing related videos...") - - related = [] of JSON::Any - - # Parse "compactVideoRenderer" items (under secondary results) - secondary_results = main_results - .dig?("secondaryResults", "secondaryResults", "results") - secondary_results.try &.as_a.each do |element| - if item = element["compactVideoRenderer"]? - related_video = parse_related_video(item) - related << JSON::Any.new(related_video) if related_video + if id != response.dig?("videoDetails", "videoId") + # YouTube may return a different video player response than expected. + # See: https://github.com/TeamNewPipe/NewPipe/issues/8713 + raise InfoException.new( + "The video returned by YouTube isn't the requested one. (#{client_config.client_type} client)" + ) + elsif playability_status == "OK" + return response + else + return nil end end - # If nothing was found previously, fall back to end screen renderer - if related.empty? - # Container for "endScreenVideoRenderer" items - player_overlays = player_response.dig?( - "playerOverlays", "playerOverlayRenderer", - "endScreen", "watchNextEndScreenRenderer", "results" - ) + def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any)) : Hash(String, JSON::Any) + # Top level elements - player_overlays.try &.as_a.each do |element| - if item = element["endScreenVideoRenderer"]? - related_video = parse_related_video(item) + main_results = player_response.dig?("contents", "twoColumnWatchNextResults") + + raise BrokenTubeException.new("twoColumnWatchNextResults") if !main_results + + # Primary results are not available on Music videos + # See: https://github.com/iv-org/invidious/pull/3238#issuecomment-1207193725 + if primary_results = main_results.dig?("results", "results", "contents") + video_primary_renderer = primary_results + .as_a.find(&.["videoPrimaryInfoRenderer"]?) + .try &.["videoPrimaryInfoRenderer"] + + video_secondary_renderer = primary_results + .as_a.find(&.["videoSecondaryInfoRenderer"]?) + .try &.["videoSecondaryInfoRenderer"] + + raise BrokenTubeException.new("videoPrimaryInfoRenderer") if !video_primary_renderer + raise BrokenTubeException.new("videoSecondaryInfoRenderer") if !video_secondary_renderer + end + + video_details = player_response.dig?("videoDetails") + if !(microformat = player_response.dig?("microformat", "playerMicroformatRenderer")) + microformat = {} of String => JSON::Any + end + + raise BrokenTubeException.new("videoDetails") if !video_details + + # Basic video infos + + title = video_details["title"]?.try &.as_s + + # We have to try to extract viewCount from videoPrimaryInfoRenderer first, + # then from videoDetails, as the latter is "0" for livestreams (we want + # to get the amount of viewers watching). + views_txt = extract_text( + video_primary_renderer + .try &.dig?("viewCount", "videoViewCountRenderer", "viewCount") + ) + views_txt ||= video_details["viewCount"]?.try &.as_s || "" + views = views_txt.gsub(/\D/, "").to_i64? + + length_txt = (microformat["lengthSeconds"]? || video_details["lengthSeconds"]) + .try &.as_s.to_i64 + + published = microformat["publishDate"]? + .try { |t| Time.parse(t.as_s, "%Y-%m-%d", Time::Location::UTC) } || Time.utc + + premiere_timestamp = microformat.dig?("liveBroadcastDetails", "startTimestamp") + .try { |t| Time.parse_rfc3339(t.as_s) } + + premiere_timestamp ||= player_response.dig?( + "playabilityStatus", "liveStreamability", + "liveStreamabilityRenderer", "offlineSlate", + "liveStreamOfflineSlateRenderer", "scheduledStartTime" + ) + .try &.as_s.to_i64 + .try { |t| Time.unix(t) } + + live_now = microformat.dig?("liveBroadcastDetails", "isLiveNow") + .try &.as_bool + live_now ||= video_details.dig?("isLive").try &.as_bool || false + + post_live_dvr = video_details.dig?("isPostLiveDvr") + .try &.as_bool || false + + # Extra video infos + + allowed_regions = microformat["availableCountries"]? + .try &.as_a.map &.as_s || [] of String + + allow_ratings = video_details["allowRatings"]?.try &.as_bool + family_friendly = microformat["isFamilySafe"]?.try &.as_bool + is_listed = video_details["isCrawlable"]?.try &.as_bool + is_upcoming = video_details["isUpcoming"]?.try &.as_bool + + keywords = video_details["keywords"]? + .try &.as_a.map &.as_s || [] of String + + # Related videos + + LOGGER.debug("extract_video_info: parsing related videos...") + + related = [] of JSON::Any + + # Parse "compactVideoRenderer" items (under secondary results) + secondary_results = main_results + .dig?("secondaryResults", "secondaryResults", "results") + secondary_results.try &.as_a.each do |element| + if item = element["compactVideoRenderer"]? + related_video = self.parse_related_video(item) related << JSON::Any.new(related_video) if related_video end end - end - # Likes - - toplevel_buttons = video_primary_renderer - .try &.dig?("videoActions", "menuRenderer", "topLevelButtons") - - if toplevel_buttons - # New Format as of december 2023 - likes_button = toplevel_buttons.dig?(0, - "segmentedLikeDislikeButtonViewModel", - "likeButtonViewModel", - "likeButtonViewModel", - "toggleButtonViewModel", - "toggleButtonViewModel", - "defaultButtonViewModel", - "buttonViewModel" - ) - - likes_button ||= toplevel_buttons.try &.as_a - .find(&.dig?("toggleButtonRenderer", "defaultIcon", "iconType").=== "LIKE") - .try &.["toggleButtonRenderer"] - - # New format as of september 2022 - likes_button ||= toplevel_buttons.try &.as_a - .find(&.["segmentedLikeDislikeButtonRenderer"]?) - .try &.dig?( - "segmentedLikeDislikeButtonRenderer", - "likeButton", "toggleButtonRenderer" + # If nothing was found previously, fall back to end screen renderer + if related.empty? + # Container for "endScreenVideoRenderer" items + player_overlays = player_response.dig?( + "playerOverlays", "playerOverlayRenderer", + "endScreen", "watchNextEndScreenRenderer", "results" ) - if likes_button - likes_txt = likes_button.dig?("accessibilityText") - # Note: The like count from `toggledText` is off by one, as it would - # represent the new like count in the event where the user clicks on "like". - likes_txt ||= (likes_button["defaultText"]? || likes_button["toggledText"]?) - .try &.dig?("accessibility", "accessibilityData", "label") - likes = likes_txt.as_s.gsub(/\D/, "").to_i64? if likes_txt - - LOGGER.trace("extract_video_info: Found \"likes\" button. Button text is \"#{likes_txt}\"") - LOGGER.debug("extract_video_info: Likes count is #{likes}") if likes - end - end - - # Description - - description = microformat.dig?("description", "simpleText").try &.as_s || "" - short_description = player_response.dig?("videoDetails", "shortDescription") - - # description_html = video_secondary_renderer.try &.dig?("description", "runs") - # .try &.as_a.try { |t| content_to_comment_html(t, video_id) } - - description_html = parse_description(video_secondary_renderer.try &.dig?("attributedDescription"), video_id) - - # Video metadata - - metadata = video_secondary_renderer - .try &.dig?("metadataRowContainer", "metadataRowContainerRenderer", "rows") - .try &.as_a - - genre = microformat["category"]? - genre_ucid = nil - license = nil - - metadata.try &.each do |row| - metadata_title = extract_text(row.dig?("metadataRowRenderer", "title")) - contents = row.dig?("metadataRowRenderer", "contents", 0) - - if metadata_title == "Category" - contents = contents.try &.dig?("runs", 0) - - genre = contents.try &.["text"]? - genre_ucid = contents.try &.dig?("navigationEndpoint", "browseEndpoint", "browseId") - elsif metadata_title == "License" - license = contents.try &.dig?("runs", 0, "text") - elsif metadata_title == "Licensed to YouTube by" - license = contents.try &.["simpleText"]? - end - end - - # Music section - - music_list = [] of VideoMusic - music_desclist = player_response.dig?( - "engagementPanels", 1, "engagementPanelSectionListRenderer", - "content", "structuredDescriptionContentRenderer", "items", 2, - "videoDescriptionMusicSectionRenderer", "carouselLockups" - ) - - music_desclist.try &.as_a.each do |music_desc| - artist = nil - album = nil - music_license = nil - - # Used when the video has multiple songs - if song_title = music_desc.dig?("carouselLockupRenderer", "videoLockup", "compactVideoRenderer", "title") - # "simpleText" for plain text / "runs" when song has a link - song = song_title["simpleText"]? || song_title.dig?("runs", 0, "text") - - # some videos can have empty tracks. See: https://www.youtube.com/watch?v=eBGIQ7ZuuiU - next if !song - end - - music_desc.dig?("carouselLockupRenderer", "infoRows").try &.as_a.each do |desc| - desc_title = extract_text(desc.dig?("infoRowRenderer", "title")) - if desc_title == "ARTIST" - artist = extract_text(desc.dig?("infoRowRenderer", "defaultMetadata")) - elsif desc_title == "SONG" - song = extract_text(desc.dig?("infoRowRenderer", "defaultMetadata")) - elsif desc_title == "ALBUM" - album = extract_text(desc.dig?("infoRowRenderer", "defaultMetadata")) - elsif desc_title == "LICENSES" - music_license = extract_text(desc.dig?("infoRowRenderer", "expandedMetadata")) + player_overlays.try &.as_a.each do |element| + if item = element["endScreenVideoRenderer"]? + related_video = self.parse_related_video(item) + related << JSON::Any.new(related_video) if related_video + end end end - music_list << VideoMusic.new(song.to_s, album.to_s, artist.to_s, music_license.to_s) - end - # Author infos + # Likes - author = video_details["author"]?.try &.as_s - ucid = video_details["channelId"]?.try &.as_s + toplevel_buttons = video_primary_renderer + .try &.dig?("videoActions", "menuRenderer", "topLevelButtons") - if author_info = video_secondary_renderer.try &.dig?("owner", "videoOwnerRenderer") - author_thumbnail = author_info.dig?("thumbnail", "thumbnails", 0, "url") - author_verified = has_verified_badge?(author_info["badges"]?) + if toplevel_buttons + # New Format as of december 2023 + likes_button = toplevel_buttons.dig?(0, + "segmentedLikeDislikeButtonViewModel", + "likeButtonViewModel", + "likeButtonViewModel", + "toggleButtonViewModel", + "toggleButtonViewModel", + "defaultButtonViewModel", + "buttonViewModel" + ) - subs_text = author_info["subscriberCountText"]? - .try { |t| t["simpleText"]? || t.dig?("runs", 0, "text") } - .try &.as_s.split(" ", 2)[0] - end + likes_button ||= toplevel_buttons.try &.as_a + .find(&.dig?("toggleButtonRenderer", "defaultIcon", "iconType").=== "LIKE") + .try &.["toggleButtonRenderer"] - # Return data + # New format as of september 2022 + likes_button ||= toplevel_buttons.try &.as_a + .find(&.["segmentedLikeDislikeButtonRenderer"]?) + .try &.dig?( + "segmentedLikeDislikeButtonRenderer", + "likeButton", "toggleButtonRenderer" + ) - if live_now - video_type = VideoType::Livestream - elsif !premiere_timestamp.nil? - video_type = VideoType::Scheduled - published = premiere_timestamp || Time.utc - else - video_type = VideoType::Video - end + if likes_button + likes_txt = likes_button.dig?("accessibilityText") + # Note: The like count from `toggledText` is off by one, as it would + # represent the new like count in the event where the user clicks on "like". + likes_txt ||= (likes_button["defaultText"]? || likes_button["toggledText"]?) + .try &.dig?("accessibility", "accessibilityData", "label") + likes = likes_txt.as_s.gsub(/\D/, "").to_i64? if likes_txt + + LOGGER.trace("extract_video_info: Found \"likes\" button. Button text is \"#{likes_txt}\"") + LOGGER.debug("extract_video_info: Likes count is #{likes}") if likes + end + end - params = { - "videoType" => JSON::Any.new(video_type.to_s), - # Basic video infos - "title" => JSON::Any.new(title || ""), - "views" => JSON::Any.new(views || 0_i64), - "likes" => JSON::Any.new(likes || 0_i64), - "lengthSeconds" => JSON::Any.new(length_txt || 0_i64), - "published" => JSON::Any.new(published.to_rfc3339), - # Extra video infos - "allowedRegions" => JSON::Any.new(allowed_regions.map { |v| JSON::Any.new(v) }), - "allowRatings" => JSON::Any.new(allow_ratings || false), - "isFamilyFriendly" => JSON::Any.new(family_friendly || false), - "isListed" => JSON::Any.new(is_listed || false), - "isUpcoming" => JSON::Any.new(is_upcoming || false), - "keywords" => JSON::Any.new(keywords.map { |v| JSON::Any.new(v) }), - "isPostLiveDvr" => JSON::Any.new(post_live_dvr), - # Related videos - "relatedVideos" => JSON::Any.new(related), # Description - "description" => JSON::Any.new(description || ""), - "descriptionHtml" => JSON::Any.new(description_html || "

    "), - "shortDescription" => JSON::Any.new(short_description.try &.as_s || nil), + + description = microformat.dig?("description", "simpleText").try &.as_s || "" + short_description = player_response.dig?("videoDetails", "shortDescription") + + # description_html = video_secondary_renderer.try &.dig?("description", "runs") + # .try &.as_a.try { |t| content_to_comment_html(t, video_id) } + + description_html = parse_description(video_secondary_renderer.try &.dig?("attributedDescription"), video_id) + # Video metadata - "genre" => JSON::Any.new(genre.try &.as_s || ""), - "genreUcid" => JSON::Any.new(genre_ucid.try &.as_s?), - "license" => JSON::Any.new(license.try &.as_s || ""), + + metadata = video_secondary_renderer + .try &.dig?("metadataRowContainer", "metadataRowContainerRenderer", "rows") + .try &.as_a + + genre = microformat["category"]? + genre_ucid = nil + license = nil + + metadata.try &.each do |row| + metadata_title = extract_text(row.dig?("metadataRowRenderer", "title")) + contents = row.dig?("metadataRowRenderer", "contents", 0) + + if metadata_title == "Category" + contents = contents.try &.dig?("runs", 0) + + genre = contents.try &.["text"]? + genre_ucid = contents.try &.dig?("navigationEndpoint", "browseEndpoint", "browseId") + elsif metadata_title == "License" + license = contents.try &.dig?("runs", 0, "text") + elsif metadata_title == "Licensed to YouTube by" + license = contents.try &.["simpleText"]? + end + end + # Music section - "music" => JSON.parse(music_list.to_json), + + music_list = [] of VideoMusic + music_desclist = player_response.dig?( + "engagementPanels", 1, "engagementPanelSectionListRenderer", + "content", "structuredDescriptionContentRenderer", "items", 2, + "videoDescriptionMusicSectionRenderer", "carouselLockups" + ) + + music_desclist.try &.as_a.each do |music_desc| + artist = nil + album = nil + music_license = nil + + # Used when the video has multiple songs + if song_title = music_desc.dig?("carouselLockupRenderer", "videoLockup", "compactVideoRenderer", "title") + # "simpleText" for plain text / "runs" when song has a link + song = song_title["simpleText"]? || song_title.dig?("runs", 0, "text") + + # some videos can have empty tracks. See: https://www.youtube.com/watch?v=eBGIQ7ZuuiU + next if !song + end + + music_desc.dig?("carouselLockupRenderer", "infoRows").try &.as_a.each do |desc| + desc_title = extract_text(desc.dig?("infoRowRenderer", "title")) + if desc_title == "ARTIST" + artist = extract_text(desc.dig?("infoRowRenderer", "defaultMetadata")) + elsif desc_title == "SONG" + song = extract_text(desc.dig?("infoRowRenderer", "defaultMetadata")) + elsif desc_title == "ALBUM" + album = extract_text(desc.dig?("infoRowRenderer", "defaultMetadata")) + elsif desc_title == "LICENSES" + music_license = extract_text(desc.dig?("infoRowRenderer", "expandedMetadata")) + end + end + music_list << VideoMusic.new(song.to_s, album.to_s, artist.to_s, music_license.to_s) + end + # Author infos - "author" => JSON::Any.new(author || ""), - "ucid" => JSON::Any.new(ucid || ""), - "authorThumbnail" => JSON::Any.new(author_thumbnail.try &.as_s || ""), - "authorVerified" => JSON::Any.new(author_verified || false), - "subCountText" => JSON::Any.new(subs_text || "-"), - } - return params -end + author = video_details["author"]?.try &.as_s + ucid = video_details["channelId"]?.try &.as_s -private def convert_url(fmt) - if cfr = fmt["signatureCipher"]?.try { |json| HTTP::Params.parse(json.as_s) } - url = URI.parse(cfr["url"]) - params = url.query_params + if author_info = video_secondary_renderer.try &.dig?("owner", "videoOwnerRenderer") + author_thumbnail = author_info.dig?("thumbnail", "thumbnails", 0, "url") + author_verified = has_verified_badge?(author_info["badges"]?) - LOGGER.debug("convert_url: Decoding '#{cfr}'") - else - url = URI.parse(fmt["url"].as_s) - params = url.query_params + subs_text = author_info["subscriberCountText"]? + .try { |t| t["simpleText"]? || t.dig?("runs", 0, "text") } + .try &.as_s.split(" ", 2)[0] + end + + # Return data + + if live_now + video_type = VideoType::Livestream + elsif !premiere_timestamp.nil? + video_type = VideoType::Scheduled + published = premiere_timestamp || Time.utc + else + video_type = VideoType::Video + end + + params = { + "videoType" => JSON::Any.new(video_type.to_s), + # Basic video infos + "title" => JSON::Any.new(title || ""), + "views" => JSON::Any.new(views || 0_i64), + "likes" => JSON::Any.new(likes || 0_i64), + "lengthSeconds" => JSON::Any.new(length_txt || 0_i64), + "published" => JSON::Any.new(published.to_rfc3339), + # Extra video infos + "allowedRegions" => JSON::Any.new(allowed_regions.map { |v| JSON::Any.new(v) }), + "allowRatings" => JSON::Any.new(allow_ratings || false), + "isFamilyFriendly" => JSON::Any.new(family_friendly || false), + "isListed" => JSON::Any.new(is_listed || false), + "isUpcoming" => JSON::Any.new(is_upcoming || false), + "keywords" => JSON::Any.new(keywords.map { |v| JSON::Any.new(v) }), + "isPostLiveDvr" => JSON::Any.new(post_live_dvr), + # Related videos + "relatedVideos" => JSON::Any.new(related), + # Description + "description" => JSON::Any.new(description || ""), + "descriptionHtml" => JSON::Any.new(description_html || "

    "), + "shortDescription" => JSON::Any.new(short_description.try &.as_s || nil), + # Video metadata + "genre" => JSON::Any.new(genre.try &.as_s || ""), + "genreUcid" => JSON::Any.new(genre_ucid.try &.as_s?), + "license" => JSON::Any.new(license.try &.as_s || ""), + # Music section + "music" => JSON.parse(music_list.to_json), + # Author infos + "author" => JSON::Any.new(author || ""), + "ucid" => JSON::Any.new(ucid || ""), + "authorThumbnail" => JSON::Any.new(author_thumbnail.try &.as_s || ""), + "authorVerified" => JSON::Any.new(author_verified || false), + "subCountText" => JSON::Any.new(subs_text || "-"), + } + + return params end - url.query_params = params - LOGGER.trace("convert_url: new url is '#{url}'") + private def convert_url(fmt) + if cfr = fmt["signatureCipher"]?.try { |json| HTTP::Params.parse(json.as_s) } + url = URI.parse(cfr["url"]) + params = url.query_params - return url.to_s -rescue ex - LOGGER.debug("convert_url: Error when parsing video URL") - LOGGER.trace(ex.inspect_with_backtrace) - return "" + LOGGER.debug("convert_url: Decoding '#{cfr}'") + else + url = URI.parse(fmt["url"].as_s) + params = url.query_params + end + + url.query_params = params + LOGGER.trace("convert_url: new url is '#{url}'") + + return url.to_s + rescue ex + LOGGER.debug("convert_url: Error when parsing video URL") + LOGGER.trace(ex.inspect_with_backtrace) + return "" + end end diff --git a/src/invidious/videos/video_preferences.cr b/src/invidious/videos/video_preferences.cr index 48177bd8..38870947 100644 --- a/src/invidious/videos/video_preferences.cr +++ b/src/invidious/videos/video_preferences.cr @@ -1,162 +1,166 @@ -struct VideoPreferences - include JSON::Serializable +module Invidious::Videos + extend self - property annotations : Bool - property preload : Bool - property autoplay : Bool - property comments : Array(String) - property continue : Bool - property continue_autoplay : Bool - property controls : Bool - property listen : Bool - property local : Bool - property preferred_captions : Array(String) - property player_style : String - property quality : String - property quality_dash : String - property raw : Bool - property region : String? - property related_videos : Bool - property speed : Float32 | Float64 - property video_end : Float64 | Int32 - property video_loop : Bool - property extend_desc : Bool - property video_start : Float64 | Int32 - property volume : Int32 - property vr_mode : Bool - property save_player_pos : Bool -end - -def process_video_params(query, preferences) - annotations = query["iv_load_policy"]?.try &.to_i? - preload = query["preload"]?.try { |q| (q == "true" || q == "1").to_unsafe } - autoplay = query["autoplay"]?.try { |q| (q == "true" || q == "1").to_unsafe } - comments = query["comments"]?.try &.split(",").map(&.downcase) - continue = query["continue"]?.try { |q| (q == "true" || q == "1").to_unsafe } - continue_autoplay = query["continue_autoplay"]?.try { |q| (q == "true" || q == "1").to_unsafe } - listen = query["listen"]?.try { |q| (q == "true" || q == "1").to_unsafe } - local = query["local"]?.try { |q| (q == "true" || q == "1").to_unsafe } - player_style = query["player_style"]? - preferred_captions = query["subtitles"]?.try &.split(",").map(&.downcase) - quality = query["quality"]? - quality_dash = query["quality_dash"]? - region = query["region"]? - related_videos = query["related_videos"]?.try { |q| (q == "true" || q == "1").to_unsafe } - speed = query["speed"]?.try &.rchop("x").to_f? - video_loop = query["loop"]?.try { |q| (q == "true" || q == "1").to_unsafe } - extend_desc = query["extend_desc"]?.try { |q| (q == "true" || q == "1").to_unsafe } - volume = query["volume"]?.try &.to_i? - vr_mode = query["vr_mode"]?.try { |q| (q == "true" || q == "1").to_unsafe } - save_player_pos = query["save_player_pos"]?.try { |q| (q == "true" || q == "1").to_unsafe } - - if preferences - # region ||= preferences.region - annotations ||= preferences.annotations.to_unsafe - preload ||= preferences.preload.to_unsafe - autoplay ||= preferences.autoplay.to_unsafe - comments ||= preferences.comments - continue ||= preferences.continue.to_unsafe - continue_autoplay ||= preferences.continue_autoplay.to_unsafe - listen ||= preferences.listen.to_unsafe - local ||= preferences.local.to_unsafe - player_style ||= preferences.player_style - preferred_captions ||= preferences.captions - quality ||= preferences.quality - quality_dash ||= preferences.quality_dash - related_videos ||= preferences.related_videos.to_unsafe - speed ||= preferences.speed - video_loop ||= preferences.video_loop.to_unsafe - extend_desc ||= preferences.extend_desc.to_unsafe - volume ||= preferences.volume - vr_mode ||= preferences.vr_mode.to_unsafe - save_player_pos ||= preferences.save_player_pos.to_unsafe - end - - annotations ||= CONFIG.default_user_preferences.annotations.to_unsafe - preload ||= CONFIG.default_user_preferences.preload.to_unsafe - autoplay ||= CONFIG.default_user_preferences.autoplay.to_unsafe - comments ||= CONFIG.default_user_preferences.comments - continue ||= CONFIG.default_user_preferences.continue.to_unsafe - continue_autoplay ||= CONFIG.default_user_preferences.continue_autoplay.to_unsafe - listen ||= CONFIG.default_user_preferences.listen.to_unsafe - local ||= CONFIG.default_user_preferences.local.to_unsafe - player_style ||= CONFIG.default_user_preferences.player_style - preferred_captions ||= CONFIG.default_user_preferences.captions - quality ||= CONFIG.default_user_preferences.quality - quality_dash ||= CONFIG.default_user_preferences.quality_dash - related_videos ||= CONFIG.default_user_preferences.related_videos.to_unsafe - speed ||= CONFIG.default_user_preferences.speed - video_loop ||= CONFIG.default_user_preferences.video_loop.to_unsafe - extend_desc ||= CONFIG.default_user_preferences.extend_desc.to_unsafe - volume ||= CONFIG.default_user_preferences.volume - vr_mode ||= CONFIG.default_user_preferences.vr_mode.to_unsafe - save_player_pos ||= CONFIG.default_user_preferences.save_player_pos.to_unsafe - - annotations = annotations == 1 - preload = preload == 1 - autoplay = autoplay == 1 - continue = continue == 1 - continue_autoplay = continue_autoplay == 1 - listen = listen == 1 - local = local == 1 - related_videos = related_videos == 1 - video_loop = video_loop == 1 - extend_desc = extend_desc == 1 - vr_mode = vr_mode == 1 - save_player_pos = save_player_pos == 1 - - if CONFIG.disabled?("dash") && quality == "dash" - quality = "high" - end - - if CONFIG.disabled?("local") && local - local = false - end - - if start = query["t"]? || query["time_continue"]? || query["start"]? - video_start = decode_time(start) - end - video_start ||= 0 - - if query["end"]? - video_end = decode_time(query["end"]) - end - video_end ||= -1 - - raw = query["raw"]?.try &.to_i? - raw ||= 0 - raw = raw == 1 - - controls = query["controls"]?.try &.to_i? - controls ||= 1 - controls = controls >= 1 - - params = VideoPreferences.new({ - annotations: annotations, - preload: preload, - autoplay: autoplay, - comments: comments, - continue: continue, - continue_autoplay: continue_autoplay, - controls: controls, - listen: listen, - local: local, - player_style: player_style, - preferred_captions: preferred_captions, - quality: quality, - quality_dash: quality_dash, - raw: raw, - region: region, - related_videos: related_videos, - speed: speed, - video_end: video_end, - video_loop: video_loop, - extend_desc: extend_desc, - video_start: video_start, - volume: volume, - vr_mode: vr_mode, - save_player_pos: save_player_pos, - }) - - return params + struct VideoPreferences + include JSON::Serializable + + property annotations : Bool + property preload : Bool + property autoplay : Bool + property comments : Array(String) + property continue : Bool + property continue_autoplay : Bool + property controls : Bool + property listen : Bool + property local : Bool + property preferred_captions : Array(String) + property player_style : String + property quality : String + property quality_dash : String + property raw : Bool + property region : String? + property related_videos : Bool + property speed : Float32 | Float64 + property video_end : Float64 | Int32 + property video_loop : Bool + property extend_desc : Bool + property video_start : Float64 | Int32 + property volume : Int32 + property vr_mode : Bool + property save_player_pos : Bool + end + + def process_video_params(query, preferences) + annotations = query["iv_load_policy"]?.try &.to_i? + preload = query["preload"]?.try { |q| (q == "true" || q == "1").to_unsafe } + autoplay = query["autoplay"]?.try { |q| (q == "true" || q == "1").to_unsafe } + comments = query["comments"]?.try &.split(",").map(&.downcase) + continue = query["continue"]?.try { |q| (q == "true" || q == "1").to_unsafe } + continue_autoplay = query["continue_autoplay"]?.try { |q| (q == "true" || q == "1").to_unsafe } + listen = query["listen"]?.try { |q| (q == "true" || q == "1").to_unsafe } + local = query["local"]?.try { |q| (q == "true" || q == "1").to_unsafe } + player_style = query["player_style"]? + preferred_captions = query["subtitles"]?.try &.split(",").map(&.downcase) + quality = query["quality"]? + quality_dash = query["quality_dash"]? + region = query["region"]? + related_videos = query["related_videos"]?.try { |q| (q == "true" || q == "1").to_unsafe } + speed = query["speed"]?.try &.rchop("x").to_f? + video_loop = query["loop"]?.try { |q| (q == "true" || q == "1").to_unsafe } + extend_desc = query["extend_desc"]?.try { |q| (q == "true" || q == "1").to_unsafe } + volume = query["volume"]?.try &.to_i? + vr_mode = query["vr_mode"]?.try { |q| (q == "true" || q == "1").to_unsafe } + save_player_pos = query["save_player_pos"]?.try { |q| (q == "true" || q == "1").to_unsafe } + + if preferences + # region ||= preferences.region + annotations ||= preferences.annotations.to_unsafe + preload ||= preferences.preload.to_unsafe + autoplay ||= preferences.autoplay.to_unsafe + comments ||= preferences.comments + continue ||= preferences.continue.to_unsafe + continue_autoplay ||= preferences.continue_autoplay.to_unsafe + listen ||= preferences.listen.to_unsafe + local ||= preferences.local.to_unsafe + player_style ||= preferences.player_style + preferred_captions ||= preferences.captions + quality ||= preferences.quality + quality_dash ||= preferences.quality_dash + related_videos ||= preferences.related_videos.to_unsafe + speed ||= preferences.speed + video_loop ||= preferences.video_loop.to_unsafe + extend_desc ||= preferences.extend_desc.to_unsafe + volume ||= preferences.volume + vr_mode ||= preferences.vr_mode.to_unsafe + save_player_pos ||= preferences.save_player_pos.to_unsafe + end + + annotations ||= CONFIG.default_user_preferences.annotations.to_unsafe + preload ||= CONFIG.default_user_preferences.preload.to_unsafe + autoplay ||= CONFIG.default_user_preferences.autoplay.to_unsafe + comments ||= CONFIG.default_user_preferences.comments + continue ||= CONFIG.default_user_preferences.continue.to_unsafe + continue_autoplay ||= CONFIG.default_user_preferences.continue_autoplay.to_unsafe + listen ||= CONFIG.default_user_preferences.listen.to_unsafe + local ||= CONFIG.default_user_preferences.local.to_unsafe + player_style ||= CONFIG.default_user_preferences.player_style + preferred_captions ||= CONFIG.default_user_preferences.captions + quality ||= CONFIG.default_user_preferences.quality + quality_dash ||= CONFIG.default_user_preferences.quality_dash + related_videos ||= CONFIG.default_user_preferences.related_videos.to_unsafe + speed ||= CONFIG.default_user_preferences.speed + video_loop ||= CONFIG.default_user_preferences.video_loop.to_unsafe + extend_desc ||= CONFIG.default_user_preferences.extend_desc.to_unsafe + volume ||= CONFIG.default_user_preferences.volume + vr_mode ||= CONFIG.default_user_preferences.vr_mode.to_unsafe + save_player_pos ||= CONFIG.default_user_preferences.save_player_pos.to_unsafe + + annotations = annotations == 1 + preload = preload == 1 + autoplay = autoplay == 1 + continue = continue == 1 + continue_autoplay = continue_autoplay == 1 + listen = listen == 1 + local = local == 1 + related_videos = related_videos == 1 + video_loop = video_loop == 1 + extend_desc = extend_desc == 1 + vr_mode = vr_mode == 1 + save_player_pos = save_player_pos == 1 + + if CONFIG.disabled?("dash") && quality == "dash" + quality = "high" + end + + if CONFIG.disabled?("local") && local + local = false + end + + if start = query["t"]? || query["time_continue"]? || query["start"]? + video_start = decode_time(start) + end + video_start ||= 0 + + if query["end"]? + video_end = decode_time(query["end"]) + end + video_end ||= -1 + + raw = query["raw"]?.try &.to_i? + raw ||= 0 + raw = raw == 1 + + controls = query["controls"]?.try &.to_i? + controls ||= 1 + controls = controls >= 1 + + params = VideoPreferences.new({ + annotations: annotations, + preload: preload, + autoplay: autoplay, + comments: comments, + continue: continue, + continue_autoplay: continue_autoplay, + controls: controls, + listen: listen, + local: local, + player_style: player_style, + preferred_captions: preferred_captions, + quality: quality, + quality_dash: quality_dash, + raw: raw, + region: region, + related_videos: related_videos, + speed: speed, + video_end: video_end, + video_loop: video_loop, + extend_desc: extend_desc, + video_start: video_start, + volume: volume, + vr_mode: vr_mode, + save_player_pos: save_player_pos, + }) + + return params + end end diff --git a/src/invidious/views/components/search_box.ecr b/src/invidious/views/components/search_box.ecr index f957c25c..1a14475a 100644 --- a/src/invidious/views/components/search_box.ecr +++ b/src/invidious/views/components/search_box.ecr @@ -1,4 +1,12 @@ +<% + search_privacy = preferences.search_privacy +%> + +<% if search_privacy %> +
    +<% else %> +<% end %>
    autofocus<% end %> diff --git a/src/invidious/views/search.ecr b/src/invidious/views/search.ecr index 2ffe27a1..ff0ced66 100644 --- a/src/invidious/views/search.ecr +++ b/src/invidious/views/search.ecr @@ -1,5 +1,10 @@ +<% + search_privacy = preferences.search_privacy + search_query = query.text.size > 30 ? HTML.escape(query.text[0,30].rstrip(".")) + "…" : HTML.escape(query.text) +%> + <% content_for "header" do %> -<%= query.text.size > 30 ? HTML.escape(query.text[0,30].rstrip(".")) + "…" : HTML.escape(query.text) %> - Invidious +<%= search_privacy ? "Search" : search_query %> - Invidious <% end %> diff --git a/src/invidious/views/user/preferences.ecr b/src/invidious/views/user/preferences.ecr index 703423c0..d6692081 100644 --- a/src/invidious/views/user/preferences.ecr +++ b/src/invidious/views/user/preferences.ecr @@ -221,6 +221,12 @@ checked<% end %>> +
    + + checked<% end %>> + <%= I18n.translate(locale, "preferences_search_privacy_description") %> +
    + <% if env.get? "user" %> <%= I18n.translate(locale, "preferences_category_subscription") %> diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr index 38c869e4..b2226e74 100644 --- a/src/invidious/yt_backend/extractors.cr +++ b/src/invidious/yt_backend/extractors.cr @@ -630,13 +630,14 @@ private module Parsers end end - # Parses an InnerTube lockupViewModel into a SearchPlaylist. + # Parses an InnerTube lockupViewModel into a SearchPlaylist or a SearchVideo # Returns nil when the given object is not a lockupViewModel. # # This structure is present since November 2024 on the "podcasts" and # "playlists" tabs of the channel page. It is usually encapsulated in either # a richItemRenderer or a richGridRenderer. # + # Since 2026-05-21, now channel videos are encapsulated in a lockupViewModel. module LockupViewModelParser extend self include BaseParser @@ -648,58 +649,185 @@ private module Parsers end private def parse_internal(item_contents, author_fallback) - playlist_id = item_contents["contentId"].as_s + content_type = item_contents["contentType"].as_s - thumbnail_view_model = item_contents.dig( - "contentImage", "thumbnailViewModel" - ) + if content_type == "LOCKUP_CONTENT_TYPE_VIDEO" + thumbnail_view_model = item_contents.dig( + "contentImage", "thumbnailViewModel" + ) + thumbnail = thumbnail_view_model.dig("image", "sources", 0, "url").as_s - thumbnail = thumbnail_view_model.dig("image", "sources", 0, "url").as_s + video_id = item_contents["contentId"].as_s - # This complicated sequences tries to extract the following data structure: - # "overlays": [{ - # "thumbnailOverlayBadgeViewModel": { - # "thumbnailBadges": [{ - # "thumbnailBadgeViewModel": { - # "text": "430 episodes", - # "badgeStyle": "THUMBNAIL_OVERLAY_BADGE_STYLE_DEFAULT" - # } - # }] - # } - # }] - # - # NOTE: this simplistic `.to_i` conversion might not work on larger - # playlists and hasn't been tested. - video_count = thumbnail_view_model.dig("overlays").as_a - .compact_map(&.dig?("thumbnailOverlayBadgeViewModel", "thumbnailBadges").try &.as_a) - .flatten - .find(nil, &.dig?("thumbnailBadgeViewModel", "text").try { |node| - {"episodes", "videos"}.any? { |str| node.as_s.ends_with?(str) } + metadata = item_contents.dig("metadata", "lockupMetadataViewModel") + title = metadata.dig("title", "content").as_s + # Contains the views of the video and the published time of the video. + metadata_parts = metadata.dig("metadata", "contentMetadataViewModel", "metadataRows", 0, "metadataParts").try &.as_a + + view_count_text = metadata_parts.try &.find { |item| item["icon"]?.nil? && item.dig?("text", "content").try &.as_s.includes?("views") } + .try &.dig("text", "content").as_s + published = metadata_parts.try &.find { |item| item["icon"]?.nil? && item.dig?("text", "content").try &.as_s.includes?("ago") } + .try { |item| decode_date(item.dig("text", "content").as_s) } || Time.local + + view_count = short_text_to_number(view_count_text || "0") + + length = thumbnail_view_model.dig("overlays", 0, "thumbnailBottomOverlayViewModel", "badges", 0, "thumbnailBadgeViewModel", "text").try &.as_s + + length_seconds = decode_length_seconds(length) if length + + return SearchVideo.new({ + title: title, + id: video_id, + author: author_fallback.name, + ucid: author_fallback.id, + published: published, + views: view_count, + description_html: "", + length_seconds: length_seconds || 0, + premiere_timestamp: Time.unix(0), + author_verified: false, + author_thumbnail: nil, + badges: VideoBadges::None, }) - .try &.dig("thumbnailBadgeViewModel", "text").as_s.to_i(strict: false) + # If it's a playlist, it's content_type would be "LOCKUP_CONTENT_TYPE_PLAYLIST" + # If it's a podcast, it's content_type would be "LOCKUP_CONTENT_TYPE_PODCAST" + # Playlist and Podcasts structures are quite similar, so we can use the same logic + # we use to parse Playlists data, for Podcasts. + else + thumbnail_view_model = item_contents.dig( + "contentImage", "collectionThumbnailViewModel", + "primaryThumbnail", "thumbnailViewModel" + ) + thumbnail = thumbnail_view_model.dig("image", "sources", 0, "url").as_s - metadata = item_contents.dig("metadata", "lockupMetadataViewModel") - title = metadata.dig("title", "content").as_s + playlist_id = item_contents["contentId"].as_s - # TODO: Retrieve "updated" info from metadata parts - # rows = metadata.dig("metadata", "contentMetadataViewModel", "metadataRows").as_a - # parts_text = rows.map(&.dig?("metadataParts", "text", "content").try &.as_s) - # One of these parts should contain a string like: "Updated 2 days ago" + # This complicated sequences tries to extract the following data structure: + # + # "overlays": [ + # { + # "thumbnailOverlayBadgeViewModel": { + # "thumbnailBadges": [ + # { + # "thumbnailBadgeViewModel": { + # "icon": { + # "sources": [ + # { + # "clientResource": { + # "imageName": "BROADCAST" + # } + # } + # ] + # }, + # "text": "5 episodes", + # "badgeStyle": "THUMBNAIL_OVERLAY_BADGE_STYLE_DEFAULT", + # "backgroundColor": { + # "lightTheme": 991526, + # "darkTheme": 991526 + # } + # } + # } + # ], + # "position": "THUMBNAIL_OVERLAY_BADGE_POSITION_BOTTOM_END" + # } + # }, + # ... <-- There is another item bellow the Object we use to extract episodes/videos + # ] + # + # NOTE: this simplistic `.to_i` conversion might not work on larger + # playlists and hasn't been tested. + video_count = thumbnail_view_model.dig("overlays").as_a + .compact_map(&.dig?("thumbnailOverlayBadgeViewModel", "thumbnailBadges").try &.as_a) + .flatten + .find(nil, &.dig?("thumbnailBadgeViewModel", "text").try { |node| + {"episodes", "videos", "lessons"}.any? { |str| node.as_s.ends_with?(str) } + }) + .try &.dig("thumbnailBadgeViewModel", "text").as_s.to_i(strict: false) - # TODO: Maybe add a button to access the first video of the playlist? - # item_contents.dig("rendererContext", "commandContext", "onTap", "innertubeCommand", "watchEndpoint") - # Available fields: "videoId", "playlistId", "params" + metadata = item_contents.dig("metadata", "lockupMetadataViewModel") + title = metadata.dig("title", "content").as_s - return SearchPlaylist.new({ - title: title, - id: playlist_id, - author: author_fallback.name, - ucid: author_fallback.id, - video_count: video_count || -1, - videos: [] of SearchPlaylistVideo, - thumbnail: thumbnail, - author_verified: false, - }) + # metadataParts is not always in the first place of the metadataRows array, therefore, + # we search for it iterating the array. We have only seen metadataRows with at least + # 2 items inside it. + # + # It looks like this: + # "metadataRows": [ + # {}, <-- empty Object + # { + # "metadataParts": [ ... ] <-- metadataParts with the information we are searching for. + # } + # ] + # + # Playlist on channels also contain metadataRows, but not with the type of data we are searching + # for which are the channel name and channel ID, instead they have two fields depending of the playlist + # updated date: + # + # It looks like this: + # "metadataRows": [ + # { + # "metadataParts": [ + # { + # "text": { + # "content": "Updated 4 days ago" + # } + # } <-- This object is missing if the playlist has not been updated in around 7 + # days + # ] + # }, + # { + # "metadataParts": [ + # { + # "text": { + # "content": "View full playlist", + # "commandRuns": [ ... ], + # "styleRuns": [ ... ]. + # } + # } <-- This object is always present, so we use this to determine if the + # metadataParts can be used or not. + # ] + # } + # ] + # + metadata_rows = metadata.dig?("metadata", "contentMetadataViewModel", "metadataRows").try &.as_a + metadata_parts = metadata_rows.try &.find { |row| + parts = row["metadataParts"]?.try &.as_a + parts && !parts.any? { |item| item.dig?("text", "content").try &.as_s == "View full playlist" } + }.try &.["metadataParts"].as_a + + if author_info = metadata_parts.try &.find(&.dig?("text", "commandRuns")) + .try &.["text"] + author = author_info["content"].as_s + author_id = author_info.dig?("commandRuns", 0, "onTap", "innertubeCommand", "browseEndpoint", "browseId") + .try &.as_s || author_fallback.id + author_verified = (author_info.dig?("attachmentRuns", 0, "element", "type", "imageType", "image", "sources", 0, "clientResource", "imageName") + .try &.as_s) == "CHECK_CIRCLE_FILLED" || false + else + author = author_fallback.name + author_id = author_fallback.id + author_verified = false + end + + # TODO: Retrieve "updated" info from metadata parts + # rows = metadata.dig("metadata", "contentMetadataViewModel", "metadataRows").as_a + # parts_text = rows.map(&.dig?("metadataParts", "text", "content").try &.as_s) + # One of these parts should contain a string like: "Updated 2 days ago" + + # TODO: Maybe add a button to access the first video of the playlist? + # item_contents.dig("rendererContext", "commandContext", "onTap", "innertubeCommand", "watchEndpoint") + # Available fields: "videoId", "playlistId", "params" + + return SearchPlaylist.new({ + title: title, + id: playlist_id, + author: author, + ucid: author_id, + video_count: video_count || -1, + videos: [] of SearchPlaylistVideo, + thumbnail: thumbnail, + author_verified: author_verified, + }) + end end def self.parser_name diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index dd709920..1783a542 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -480,7 +480,7 @@ module YoutubeAPI # # ``` # # Valid channel "brand URL" gives the related UCID and browse ID - # channel_a = YoutubeAPI.resolve_url("https://youtube.com/c/google") + # channel_a = YoutubeAPI.resolve_url("https://www.youtube.com/c/google") # channel_a # => { # "endpoint": { # "browseEndpoint": { @@ -492,7 +492,7 @@ module YoutubeAPI # } # # # Invalid URL returns throws an InfoException - # channel_b = YoutubeAPI.resolve_url("https://youtube.com/c/invalid") + # channel_b = YoutubeAPI.resolve_url("https://www.youtube.com/c/invalid") # ``` # def resolve_url(url : String, client_config : ClientConfig | Nil = nil)