Merge remote-tracking branch 'origin/master' into oidc-support

src/invidious.cr

@@ -67,20 +67,9 @@ rescue ex
   puts "Check your 'config.yml' database settings or PostgreSQL settings."
   exit(1)
 end
-ARCHIVE_URL = URI.parse("https://archive.org")
-PUBSUB_URL  = URI.parse("https://pubsubhubbub.appspot.com")
-REDDIT_URL  = URI.parse("https://www.reddit.com")
-YT_URL      = URI.parse("https://www.youtube.com")
-HOST_URL    = make_host_url(Kemal.config)
-
-CHARS_SAFE         = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"
-TEST_IDS           = {"AgbeGFYluEA", "BaW_jenozKc", "a9LDPn-MO4I", "ddFvjfvPnqk", "iqKdEhx-dD4"}
+HOST_URL           = make_host_url(Kemal.config)
 MAX_ITEMS_PER_PAGE = 1500
 
-REQUEST_HEADERS_WHITELIST  = {"accept", "accept-encoding", "cache-control", "content-length", "if-none-match", "range"}
-RESPONSE_HEADERS_BLACKLIST = {"access-control-allow-origin", "alt-svc", "server", "cross-origin-opener-policy-report-only", "report-to", "cross-origin", "timing-allow-origin", "cross-origin-resource-policy"}
-HTTP_CHUNK_SIZE            = 10485760 # ~10MB
-
 CURRENT_BRANCH  = {{ "#{`git branch | sed -n '/* /s///p'`.strip}" }}
 CURRENT_COMMIT  = {{ "#{`git rev-list HEAD --max-count=1 --abbrev-commit`.strip}" }}
 CURRENT_VERSION = {{ "#{`git log -1 --format=%ci | awk '{print $1}' | sed s/-/./g`.strip}" }}
@@ -97,7 +86,7 @@ SOFTWARE = {
   "branch"  => "#{CURRENT_BRANCH}",
 }
 
-YT_POOL = YoutubeConnectionPool.new(YT_URL, capacity: CONFIG.pool_size)
+YT_POOL = YoutubeConnectionPool.new(URI.parse("https://www.youtube.com"), capacity: CONFIG.pool_size)
 
 # Image request pool
 

src/invidious/channels/about.cr

@@ -12,6 +12,7 @@ record AboutChannel,
   sub_count : Int32,
   joined : Time,
   is_family_friendly : Bool,
+  pronouns : String?,
   allowed_regions : Array(String),
   tabs : Array(String),
   tags : Array(String),
@@ -160,14 +161,21 @@ def get_about_info(ucid, locale) : AboutChannel
   end
 
   sub_count = 0
+  pronouns = nil
 
   if (metadata_rows = initdata.dig?("header", "pageHeaderRenderer", "content", "pageHeaderViewModel", "metadata", "contentMetadataViewModel", "metadataRows").try &.as_a)
     metadata_rows.each do |row|
-      metadata_part = row.dig?("metadataParts").try &.as_a.find { |i| i.dig?("text", "content").try &.as_s.includes?("subscribers") }
-      if !metadata_part.nil?
-        sub_count = short_text_to_number(metadata_part.dig("text", "content").as_s.split(" ")[0]).to_i32
+      subscribe_metadata_part = row.dig?("metadataParts").try &.as_a.find { |i| i.dig?("text", "content").try &.as_s.includes?("subscribers") }
+      if !subscribe_metadata_part.nil?
+        sub_count = short_text_to_number(subscribe_metadata_part.dig("text", "content").as_s.split(" ")[0]).to_i32
       end
-      break if sub_count != 0
+
+      pronoun_metadata_part = row.dig?("metadataParts").try &.as_a.find { |i| i.dig?("tooltip").try &.as_s.includes?("Pronouns") }
+      if !pronoun_metadata_part.nil?
+        pronouns = pronoun_metadata_part.dig("text", "content").as_s
+      end
+
+      break if sub_count != 0 && !pronouns.nil?
     end
   end
 
@@ -184,6 +192,7 @@ def get_about_info(ucid, locale) : AboutChannel
     sub_count: sub_count,
     joined: joined,
     is_family_friendly: is_family_friendly,
+    pronouns: pronouns,
     allowed_regions: allowed_regions,
     tabs: tab_names,
     tags: tags,

src/invidious/comments/reddit.cr

@@ -1,5 +1,6 @@
 module Invidious::Comments
   extend self
+  private REDDIT_URL = URI.parse("https://www.reddit.com")
 
   def fetch_reddit(id, sort_by = "confidence")
     client = make_client(REDDIT_URL)

src/invidious/frontend/watch_page.cr

@@ -36,7 +36,7 @@ module Invidious::Frontend::WatchPage
     return String.build(4000) do |str|
       str << "<form"
       str << " class=\"pure-form pure-form-stacked\""
-      str << " action='#{url}'"
+      str << " action='" << HTML.escape(url) << "'"
       str << " method='post'"
       str << " rel='noopener noreferrer'"
       str << " target='_blank'>"

src/invidious/helpers/helpers.cr

@@ -1,5 +1,7 @@
 require "./macros"
 
+TEST_IDS = {"AgbeGFYluEA", "BaW_jenozKc", "a9LDPn-MO4I", "ddFvjfvPnqk", "iqKdEhx-dD4"}
+
 struct Nonce
   include DB::Serializable
 

src/invidious/helpers/utils.cr

@@ -1,3 +1,5 @@
+PUBSUB_URL = URI.parse("https://pubsubhubbub.appspot.com")
+
 # See http://www.evanmiller.org/how-not-to-sort-by-average-rating.html
 def ci_lower_bound(pos, n)
   if n == 0

src/invidious/playlists.cr

@@ -107,7 +107,11 @@ struct Playlist
 
       json.field "author", self.author
       json.field "authorId", self.ucid
-      json.field "authorUrl", "/channel/#{self.ucid}"
+      if !self.ucid.empty?
+        json.field "authorUrl", "/channel/#{self.ucid}"
+      else
+        json.field "authorUrl", ""
+      end
       json.field "subtitle", self.subtitle
 
       json.field "authorThumbnails" do

src/invidious/routes/api/v1/channels.cr

@@ -104,6 +104,7 @@ module Invidious::Routes::API::V1::Channels
         json.field "tabs", channel.tabs
         json.field "tags", channel.tags
         json.field "authorVerified", channel.verified
+        json.field "pronouns", channel.pronouns
 
         json.field "latestVideos" do
           json.array do

src/invidious/routes/api/v1/videos.cr

@@ -1,6 +1,9 @@
 require "html"
 
 module Invidious::Routes::API::V1::Videos
+  private INTERNET_ARCHIVE_URL = URI.parse("https://archive.org")
+  private CHARS_SAFE           = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"
+
   def self.videos(env)
     locale = env.get("preferences").as(Preferences).locale
 
@@ -279,7 +282,7 @@ module Invidious::Routes::API::V1::Videos
 
         file = URI.encode_www_form("#{id[0, 3]}/#{id}.xml")
 
-        location = make_client(ARCHIVE_URL, &.get("/download/youtubeannotations_#{index}/#{id[0, 2]}.tar/#{file}"))
+        location = make_client(INTERNET_ARCHIVE_URL, &.get("/download/youtubeannotations_#{index}/#{id[0, 2]}.tar/#{file}"))
 
         if !location.headers["Location"]?
           env.response.status_code = location.status_code

src/invidious/routes/before_all.cr

@@ -1,4 +1,16 @@
 module Invidious::Routes::BeforeAll
+  struct CompanionCSP
+    property companion_urls : String = ""
+
+    def initialize
+      self.companion_urls = CONFIG.invidious_companion.reject(&.builtin_proxy).map do |companion|
+        "#{companion.public_url.scheme}://#{companion.public_url.host}#{companion.public_url.port ? ":#{companion.public_url.port}" : ""}"
+      end.join(" ")
+    end
+  end
+
+  private COMPANION_CSP = CompanionCSP.new
+
   def self.handle(env)
     preferences = Preferences.from_json("{}")
 
@@ -35,9 +47,9 @@ module Invidious::Routes::BeforeAll
       "style-src 'self' 'unsafe-inline'",
       "img-src 'self' data:",
       "font-src 'self' data:",
-      "connect-src 'self'",
+      "connect-src 'self' " + COMPANION_CSP.companion_urls,
       "manifest-src 'self'",
-      "media-src 'self' blob:",
+      "media-src 'self' blob: " + COMPANION_CSP.companion_urls,
       "child-src 'self' blob:",
       "frame-src 'self'",
       "frame-ancestors " + frame_ancestors,

src/invidious/routes/embed.cr

@@ -208,17 +208,6 @@ module Invidious::Routes::Embed
 
     if CONFIG.invidious_companion.present?
       invidious_companion = CONFIG.invidious_companion.sample
-      invidious_companion_urls = CONFIG.invidious_companion.reject(&.builtin_proxy).map do |companion|
-        uri =
-          "#{companion.public_url.scheme}://#{companion.public_url.host}#{companion.public_url.port ? ":#{companion.public_url.port}" : ""}"
-      end.join(" ")
-
-      if !invidious_companion_urls.empty?
-        env.response.headers["Content-Security-Policy"] =
-          env.response.headers["Content-Security-Policy"]
-            .gsub("media-src", "media-src #{invidious_companion_urls}")
-            .gsub("connect-src", "connect-src #{invidious_companion_urls}")
-      end
     end
 
     rendered "embed"

src/invidious/routes/routes.cr

@@ -0,0 +1,20 @@
+module Invidious::Routes
+  private REQUEST_HEADERS_WHITELIST = {
+    "accept",
+    "accept-encoding",
+    "cache-control",
+    "content-length",
+    "if-none-match",
+    "range",
+  }
+  private RESPONSE_HEADERS_BLACKLIST = {
+    "access-control-allow-origin",
+    "alt-svc",
+    "server",
+    "cross-origin-opener-policy-report-only",
+    "report-to",
+    "cross-origin",
+    "timing-allow-origin",
+    "cross-origin-resource-policy",
+  }
+end

src/invidious/routes/video_playback.cr

@@ -1,4 +1,6 @@
 module Invidious::Routes::VideoPlayback
+  private HTTP_CHUNK_SIZE = 10485760 # ~10MB
+
   # /videoplayback
   def self.get_video_playback(env)
     locale = env.get("preferences").as(Preferences).locale

src/invidious/routes/watch.cr

@@ -193,17 +193,6 @@ module Invidious::Routes::Watch
 
     if CONFIG.invidious_companion.present?
       invidious_companion = CONFIG.invidious_companion.sample
-      invidious_companion_urls = CONFIG.invidious_companion.reject(&.builtin_proxy).map do |companion|
-        uri =
-          "#{companion.public_url.scheme}://#{companion.public_url.host}#{companion.public_url.port ? ":#{companion.public_url.port}" : ""}"
-      end.join(" ")
-
-      if !invidious_companion_urls.empty?
-        env.response.headers["Content-Security-Policy"] =
-          env.response.headers["Content-Security-Policy"]
-            .gsub("media-src", "media-src #{invidious_companion_urls}")
-            .gsub("connect-src", "connect-src #{invidious_companion_urls}")
-      end
     end
 
     templated "watch"

src/invidious/search/filters.cr

@@ -57,8 +57,6 @@ module Invidious::Search
     # Values correspond to { "1:varint": <X> }
     enum Sort
       Relevance = 0
-      Rating    = 1
-      Date      = 2
       Views     = 3
     end
 

src/invidious/user/imports.cr

@@ -30,28 +30,24 @@ struct Invidious::User
       return subscriptions
     end
 
-    def parse_playlist_export_csv(user : User, raw_input : String)
+    # Parse a CSV Google Takeout - Youtube Playlist file
+    def parse_playlist_export_csv(user : User, playlist_name : String, raw_input : String)
       # Split the input into head and body content
-      raw_head, raw_body = raw_input.strip('\n').split("\n\n", limit: 2, remove_empty: true)
+      raw_head, raw_body = raw_input.split("\n", limit: 2, remove_empty: true)
 
       # Create the playlist from the head content
       csv_head = CSV.new(raw_head.strip('\n'), headers: true)
       csv_head.next
-      title = csv_head[4]
-      description = csv_head[5]
-      visibility = csv_head[6]
+      title = playlist_name
 
-      if visibility.compare("Public", case_insensitive: true) == 0
-        privacy = PlaylistPrivacy::Public
-      else
-        privacy = PlaylistPrivacy::Private
-      end
+      description = "This is the default description of an imported playlist. Feel Free to change it as you see fit."
+      privacy = PlaylistPrivacy::Private
 
       playlist = create_playlist(title, privacy, user)
       Invidious::Database::Playlists.update_description(playlist.id, description)
 
       # Add each video to the playlist from the body content
-      csv_body = CSV.new(raw_body.strip('\n'), headers: true)
+      csv_body = CSV.new(raw_body.strip('\n'), headers: false)
       csv_body.each do |row|
         video_id = row[0]
         if playlist
@@ -204,10 +200,12 @@ struct Invidious::User
     end
 
     def from_youtube_pl(user : User, body : String, filename : String, type : String) : Bool
-      extension = filename.split(".").last
+      filename_array = filename.split(".")
+      playlist_name = filename_array.first
+      extension = filename_array.last
 
       if extension == "csv" || type == "text/csv"
-        playlist = parse_playlist_export_csv(user, body)
+        playlist = parse_playlist_export_csv(user, playlist_name, body)
         if playlist
           return true
         else

src/invidious/views/components/channel_info.ecr

@@ -12,7 +12,10 @@
     <div class="pure-u-1-2 flex-left flexible">
         <div class="channel-profile">
             <img src="/ggpht<%= channel_profile_pic %>" alt="" />
-            <span class="channel-name"><%= author %></span><% if !channel.verified.nil? && channel.verified %>&nbsp;<i class="icon ion ion-md-checkmark-circle"></i><% end %>
+            <div class="channel-name-pronouns">
+                <span class="channel-name"><%= author %></span><% if !channel.verified.nil? && channel.verified %>&nbsp;<i class="icon ion ion-md-checkmark-circle"></i><% end %>
+                <% if !channel.pronouns.nil? %><br /><span class="channel-pronouns"><%= channel.pronouns %></span><% end %>
+            </div>
         </div>
     </div>
 

src/invidious/views/components/player.ecr

@@ -25,7 +25,7 @@
                audio_streams.each_with_index do |fmt, i|
                 src_url  = "/latest_version?id=#{video.id}&itag=#{fmt["itag"]}"
                 src_url += "&local=true" if params.local
-                src_url = invidious_companion.public_url.to_s + src_url + 
+                src_url = invidious_companion.public_url.to_s + src_url +
                             "&check=#{invidious_companion_check_id}" if (invidious_companion)
 
                 bitrate = fmt["bitrate"]

src/invidious/views/template.ecr

@@ -159,7 +159,7 @@
                             <% end %>
                             @ <%= CURRENT_BRANCH %>
                             <% if CURRENT_TAG != "" %>
-                            (                                
+                            (
                                 <% if CONFIG.modified_source_code_url %>
                                     <a href="<%= CONFIG.modified_source_code_url %>/releases/tag/<%= CURRENT_TAG %>"><%= CURRENT_TAG %></a>
                                 <% else %>

src/invidious/views/watch.ecr

@@ -79,11 +79,11 @@ we're going to need to do it here in order to allow for translations.
     <h1>
         <%= title %>
         <% if params.listen %>
-            <a title="<%=translate(locale, "Video mode")%>" href="/watch?<%= env.params.query %>&listen=0">
+            <a title="<%=translate(locale, "Video mode")%>" id="link-iv-listen" data-base-url="/watch?<%= env.params.query %>&listen=0" href="/watch?<%= env.params.query %>&listen=0">
                 <i class="icon ion-ios-videocam"></i>
             </a>
         <% else %>
-            <a title="<%=translate(locale, "Audio mode")%>" href="/watch?<%= env.params.query %>&listen=1">
+            <a title="<%=translate(locale, "Audio mode")%>" id="link-iv-listen" data-base-url="/watch?<%= env.params.query %>&listen=1" href="/watch?<%= env.params.query %>&listen=1">
                 <i class="icon ion-md-headset"></i>
             </a>
         <% end %>
@@ -125,7 +125,7 @@ we're going to need to do it here in order to allow for translations.
                     end
                 -%>
                 <a id="link-yt-watch" rel="noreferrer noopener" data-base-url="<%= link_yt_watch %>" href="<%= link_yt_watch %>"><%= translate(locale, "videoinfo_watch_on_youTube") %></a>
-                (<a id="link-yt-embed" rel="noreferrer noopener" data-base-url="<%= link_yt_embed %>" href="<%= link_yt_embed %>"><%= translate(locale, "videoinfo_youTube_embed_link") %></a>)
+                (<a id="link-yt-embed" rel="noopener" referrerpolicy="origin-when-cross-origin" data-base-url="<%= link_yt_embed %>" href="<%= link_yt_embed %>"><%= translate(locale, "videoinfo_youTube_embed_link") %></a>)
             </span>
 
             <p id="watch-on-another-invidious-instance">