Merge branch 'master' into oidc-support

src/ext/kemal_static_file_handler.cr

@@ -1,3 +1,24 @@
+{% if compare_versions(Crystal::VERSION, "1.17.0-dev") >= 0 %}
+  # Strip StaticFileHandler from the binary
+  #
+  # This allows us to compile on 1.17.0 as the compiler won't try to
+  # semantically check the outdated upstream code.
+  class Kemal::Config
+    private def setup_static_file_handler
+    end
+  end
+
+  # Nullify `Kemal::StaticFileHandler`
+  #
+  # Needed until the next release of Kemal after 1.7
+  class Kemal::StaticFileHandler < HTTP::StaticFileHandler
+    def call(context : HTTP::Server::Context)
+    end
+  end
+
+  {% skip_file %}
+{% end %}
+
 # Since systems have a limit on number of open files (`ulimit -a`),
 # we serve them from memory to avoid 'Too many open files' without needing
 # to modify ulimit.

src/invidious.cr

@@ -78,7 +78,7 @@ TEST_IDS           = {"AgbeGFYluEA", "BaW_jenozKc", "a9LDPn-MO4I", "ddFvjfvPnqk"
 MAX_ITEMS_PER_PAGE = 1500
 
 REQUEST_HEADERS_WHITELIST  = {"accept", "accept-encoding", "cache-control", "content-length", "if-none-match", "range"}
-RESPONSE_HEADERS_BLACKLIST = {"access-control-allow-origin", "alt-svc", "server"}
+RESPONSE_HEADERS_BLACKLIST = {"access-control-allow-origin", "alt-svc", "server", "cross-origin-opener-policy-report-only", "report-to", "cross-origin", "timing-allow-origin", "cross-origin-resource-policy"}
 HTTP_CHUNK_SIZE            = 10485760 # ~10MB
 
 CURRENT_BRANCH  = {{ "#{`git branch | sed -n '/* /s///p'`.strip}" }}
@@ -223,19 +223,25 @@ error 500 do |env, exception|
   error_template(500, exception)
 end
 
-static_headers do |env|
-  env.response.headers.add("Cache-Control", "max-age=2629800")
-end
-
 # Init Kemal
 
-public_folder "assets"
-
 Kemal.config.powered_by_header = false
 add_handler FilteredCompressHandler.new
 add_handler APIHandler.new
 add_handler AuthHandler.new
 add_handler DenyFrame.new
+
+{% if compare_versions(Crystal::VERSION, "1.17.0-dev") >= 0 %}
+  Kemal.config.serve_static = false
+  add_handler Invidious::HttpServer::StaticAssetsHandler.new("assets", directory_listing: false)
+{% else %}
+  public_folder "assets"
+
+  static_headers do |env|
+    env.response.headers.add("Cache-Control", "max-age=2629800")
+  end
+{% end %}
+
 add_context_storage_type(Array(String))
 add_context_storage_type(Preferences)
 add_context_storage_type(Invidious::User)

src/invidious/helpers/crystal_class_overrides.cr

@@ -3,15 +3,28 @@
 # IPv6 addresses.
 #
 class TCPSocket
-  def initialize(host, port, dns_timeout = nil, connect_timeout = nil, blocking = false, family = Socket::Family::UNSPEC)
-    Addrinfo.tcp(host, port, timeout: dns_timeout, family: family) do |addrinfo|
-      super(addrinfo.family, addrinfo.type, addrinfo.protocol, blocking)
-      connect(addrinfo, timeout: connect_timeout) do |error|
-        close
-        error
+  {% if compare_versions(Crystal::VERSION, "1.18.0-dev") >= 0 %}
+    def initialize(host : String, port, dns_timeout = nil, connect_timeout = nil, blocking = false, family = Socket::Family::UNSPEC)
+      Addrinfo.tcp(host, port, timeout: dns_timeout, family: family) do |addrinfo|
+        super(family: addrinfo.family, type: addrinfo.type, protocol: addrinfo.protocol)
+        Socket.set_blocking(self.fd, blocking)
+        connect(addrinfo, timeout: connect_timeout) do |error|
+          close
+          error
+        end
       end
     end
-  end
+  {% else %}
+    def initialize(host : String, port, dns_timeout = nil, connect_timeout = nil, blocking = false, family = Socket::Family::UNSPEC)
+      Addrinfo.tcp(host, port, timeout: dns_timeout, family: family) do |addrinfo|
+        super(addrinfo.family, addrinfo.type, addrinfo.protocol, blocking)
+        connect(addrinfo, timeout: connect_timeout) do |error|
+          close
+          error
+        end
+      end
+    end
+  {% end %}
 end
 
 # :ditto:

src/invidious/http_server/static_assets_handler.cr

@@ -0,0 +1,120 @@
+{% skip_file if compare_versions(Crystal::VERSION, "1.17.0-dev") < 0 %}
+
+module Invidious::HttpServer
+  class StaticAssetsHandler < HTTP::StaticFileHandler
+    # In addition to storing the actual data of a file, it also implements the required
+    # getters needed for the object to imitate a `File::Stat` within `StaticFileHandler`.
+    #
+    # Since the `File::Stat` is created once in `#call` and then passed around to the
+    # rest of the class's methods, imitating the object allows us to only lookup
+    # the cache hash once for every request.
+    #
+    private record CachedFile, data : Bytes, size : Int64, modification_time : Time do
+      def directory?
+        false
+      end
+
+      def file?
+        true
+      end
+    end
+
+    CACHE_LIMIT = 5_000_000 # 5MB
+    @@current_cache_size = 0
+    @@cached_files = {} of Path => CachedFile
+
+    # Returns metadata for the requested file
+    #
+    # If the requested file is cached, a `CachedFile` is returned instead of a `File::Stat`.
+    # This represents the metadata info of a cached file and implements all the methods of `File::Stat` that
+    # is used by the `StaticAssetsHandler`.
+    #
+    # The `CachedFile` also stores the raw bytes of the cached file, and this method serves as the place where
+    # the cached file is retrieved if it exists. Though the data will only be read in `#serve_file`
+    private def file_info(expanded_path : Path)
+      file_path = @public_dir.join(expanded_path.to_kind(Path::Kind.native))
+      {@@cached_files[file_path]? || File.info?(file_path), file_path}
+    end
+
+    # Add "Cache-Control" header to the response
+    private def add_cache_headers(response_headers : HTTP::Headers, last_modified : Time) : Nil
+      super; response_headers["Cache-Control"] = "max-age=2629800"
+    end
+
+    # Serves and caches the file at the given path.
+    #
+    # This is an override of `serve_file` to allow serving a file from memory, and to cache it
+    # it as needed.
+    private def serve_file(context : HTTP::Server::Context, file_info, file_path : Path, original_file_path : Path, last_modified : Time)
+      context.response.content_type = MIME.from_filename(original_file_path.to_s, "application/octet-stream")
+
+      range_header = context.request.headers["Range"]?
+
+      # If the file is cached we can just directly serve it
+      if file_info.is_a? CachedFile
+        return dispatch_serve(context, file_info.data, file_info, range_header)
+      end
+
+      # Otherwise we'll need to read from disk and cache it
+      retrieve_bytes_from = IO::Memory.new
+      File.open(file_path) do |file|
+        # We cannot cache partial data so we'll rewind and read from the start
+        if range_header
+          dispatch_serve(context, file, file_info, range_header)
+          IO.copy(file.rewind, retrieve_bytes_from)
+        else
+          context.response.output = IO::MultiWriter.new(context.response.output, retrieve_bytes_from, sync_close: true)
+          dispatch_serve(context, file, file_info, range_header)
+        end
+      end
+
+      return flush_io_to_cache(retrieve_bytes_from, file_path, file_info)
+    end
+
+    # Writes file data to the cache
+    private def flush_io_to_cache(io, file_path, file_info)
+      if (@@current_cache_size += file_info.size) <= CACHE_LIMIT
+        @@cached_files[file_path] = CachedFile.new(io.to_slice, file_info.size, file_info.modification_time)
+      end
+    end
+
+    # Either send the file in full, or just fragments of it depending on the request
+    private def dispatch_serve(context, file, file_info, range_header)
+      if range_header
+        # an IO is needed for `serve_file_range`
+        file = file.is_a?(Bytes) ? IO::Memory.new(file, writeable: false) : file
+        serve_file_range(context, file, range_header, file_info)
+      else
+        context.response.headers["Accept-Ranges"] = "bytes"
+        serve_file_full(context, file, file_info)
+      end
+    end
+
+    # If we're serving the full file right away then there's no need for an IO at all.
+    private def serve_file_full(context : HTTP::Server::Context, file : Bytes, file_info)
+      context.response.status = :ok
+      context.response.content_length = file_info.size
+      context.response.write file
+    end
+
+    # Serves segments of a file based on the `Range header`
+    #
+    # An override of `serve_file_range` to allow using a generic IO rather than a `File`.
+    # Literally the same code as what we inherited but just with the `file` argument's type
+    # being set to `IO` rather than `File`
+    #
+    # Can be removed once https://github.com/crystal-lang/crystal/issues/15817 is fixed.
+    private def serve_file_range(context : HTTP::Server::Context, file : IO, range_header : String, file_info)
+      # Paste in the body of inherited serve_file_range
+      {{@type.superclass.methods.select(&.name.==("serve_file_range"))[0].body}}
+    end
+
+    # Clear cached files.
+    #
+    # This is only used in the specs to clear the cache before each handler test
+    def self.clear_cache
+      @@current_cache_size = 0
+      return @@cached_files.clear
+    end
+  end
+end

src/invidious/routes/companion.cr

@@ -1,5 +1,5 @@
 module Invidious::Routes::Companion
-  # /companion
+  # GET /companion
   def self.get_companion(env)
     url = env.request.path
     if env.request.query
@@ -16,6 +16,23 @@ module Invidious::Routes::Companion
     end
   end
 
+  # POST /companion
+  def self.post_companion(env)
+    url = env.request.path
+    if env.request.query
+      url += "?#{env.request.query}"
+    end
+
+    begin
+      COMPANION_POOL.client do |wrapper|
+        wrapper.client.post(url, env.request.headers, env.request.body) do |resp|
+          return self.proxy_companion(env, resp)
+        end
+      end
+    rescue ex
+    end
+  end
+
   def self.options_companion(env)
     url = env.request.path
     if env.request.query

src/invidious/routing.cr

@@ -228,6 +228,7 @@ module Invidious::Routing
   def register_companion_routes
     if CONFIG.invidious_companion.present?
       get "/companion/*", Routes::Companion, :get_companion
+      post "/companion/*", Routes::Companion, :post_companion
       options "/companion/*", Routes::Companion, :options_companion
     end
   end

src/invidious/views/components/player.ecr

@@ -1,3 +1,6 @@
+<%
+    invidious_companion_check_id = invidious_companion_encrypt(video.id) if invidious_companion
+%>
 <video style="outline:none;width:100%;background-color:#000" playsinline poster="<%= thumbnail %>"
     id="player" class="on-video_player video-js player-style-<%= params.player_style %>"
     preload="<% if params.preload %>auto<% else %>none<% end %>"
@@ -23,7 +26,7 @@
                 src_url  = "/latest_version?id=#{video.id}&itag=#{fmt["itag"]}"
                 src_url += "&local=true" if params.local
                 src_url = invidious_companion.public_url.to_s + src_url + 
-                            "&check=#{invidious_companion_encrypt(video.id)}" if (invidious_companion)
+                            "&check=#{invidious_companion_check_id}" if (invidious_companion)
 
                 bitrate = fmt["bitrate"]
                 mimetype = HTML.escape(fmt["mimeType"].as_s)
@@ -39,7 +42,7 @@
             <% if params.quality == "dash"
                src_url = "/api/manifest/dash/id/" + video.id + "?local=true&unique_res=1"
                src_url = invidious_companion.public_url.to_s + src_url +
-                            "&check=#{invidious_companion_encrypt(video.id)}" if (invidious_companion)
+                            "&check=#{invidious_companion_check_id}" if (invidious_companion)
             %>
                 <source src="<%= src_url %>" type='application/dash+xml' label="dash">
             <% end %>
@@ -51,7 +54,7 @@
                 src_url  = "/latest_version?id=#{video.id}&itag=#{fmt["itag"]}"
                 src_url += "&local=true" if params.local
                 src_url = invidious_companion.public_url.to_s + src_url +
-                            "&check=#{invidious_companion_encrypt(video.id)}" if (invidious_companion)
+                            "&check=#{invidious_companion_check_id}" if (invidious_companion)
 
                 quality = fmt["quality"]
                 mimetype = HTML.escape(fmt["mimeType"].as_s)
@@ -68,15 +71,17 @@
         <% preferred_captions.each do |caption|
             api_captions_url = "/api/v1/captions/"
             api_captions_url = invidious_companion.public_url.to_s + api_captions_url if (invidious_companion)
+            api_captions_check_id = "&check=#{invidious_companion_check_id}"
         %>
-            <track kind="captions" src="<%= api_captions_url %><%= video.id %>?label=<%= caption.name %>" label="<%= caption.name %>">
+            <track kind="captions" src="<%= api_captions_url %><%= video.id %>?label=<%= caption.name %><%= api_captions_check_id %>" label="<%= caption.name %>">
         <% end %>
 
         <% captions.each do |caption|
             api_captions_url = "/api/v1/captions/"
             api_captions_url = invidious_companion.public_url.to_s + api_captions_url if (invidious_companion)
+            api_captions_check_id = "&check=#{invidious_companion_check_id}"
         %>
-            <track kind="captions" src="<%= api_captions_url %><%= video.id %>?label=<%= caption.name %>" label="<%= caption.name %>">
+            <track kind="captions" src="<%= api_captions_url %><%= video.id %>?label=<%= caption.name %><%= api_captions_check_id %>" label="<%= caption.name %>">
         <% end %>
     <% end %>
 </video>