Merge branch 'develop' into hellthread-filter-fix

config/dev.exs

@@ -16,7 +16,8 @@
   debug_errors: true,
   code_reloader: true,
   check_origin: false,
-  watchers: []
+  watchers: [],
+  secure_cookie_flag: false
 
 config :pleroma, Pleroma.Mailer, adapter: Swoosh.Adapters.Local
 

docs/Clients.md

@@ -26,60 +26,71 @@ Feel free to contact us to be added to this list!
 - Source Code: <https://github.com/ReticentJohn/Amaroq>
 - Contact: [@eurasierboy@mastodon.social](https://mastodon.social/users/eurasierboy)
 - Platforms: iOS
+- Features: No Streaming
 
 ### Nekonium
 - Homepage: [F-Droid Repository](https://repo.gdgd.jp.net/), [Google Play](https://play.google.com/store/apps/details?id=com.apps.nekonium), [Amazon](https://www.amazon.co.jp/dp/B076FXPRBC/)
 - Source: <https://git.gdgd.jp.net/lin/nekonium/>
 - Contact: [@lin@pleroma.gdgd.jp.net](https://pleroma.gdgd.jp.net/users/lin)
 - Platforms: Android
+- Features: Streaming Ready
 
 ### Mastalab
 - Source Code: <https://gitlab.com/tom79/mastalab/>
 - Contact: [@tom79@mastodon.social](https://mastodon.social/users/tom79)
 - Platforms: Android
+- Features: Streaming Ready
 
 ### Roma
 - Homepage: <http://www.pleroma.com/>
 - Source Code: ???
 - Platforms: iOS, Android
+- Features: No Streaming
 
 ### Tootdon
 - Homepage: <http://tootdon.club/>, <http://blog.mastodon-tootdon.com/>
 - Source Code: ???
 - Contact: [@tootdon@mstdn.jp](https://mstdn.jp/users/tootdon)
 - Platforms: Android, iOS
+- Features: No Streaming
 
 ### Tusky
 - Homepage: <https://tuskyapp.github.io/>
 - Source Code: <https://github.com/tuskyapp/Tusky>
 - Contact: [@ConnyDuck@mastodon.social](https://mastodon.social/users/ConnyDuck)
 - Platforms: Android
+- Features: No Streaming
 
 ### Twidere
 - Homepage: <https://twidere.mariotaku.org/>
 - Source Code: <https://github.com/TwidereProject/Twidere-Android/>, <https://github.com/TwidereProject/Twidere-iOS/>
 - Contact: <me@mariotaku.org>
 - Platform: Android, iOS
+- Features: No Streaming
 
 ## Alternative Web Interfaces
 ### Brutaldon
 - Homepage: <https://jfm.carcosa.net/projects/software/brutaldon/>
 - Source Code: <https://github.com/jfmcbrayer/brutaldon>
 - Contact: [@gcupc@glitch.social](https://glitch.social/users/gcupc)
+- Features: No Streaming
 
 ### Feather
 - Source Code: <https://github.com/kaniini/feather>
 - Contact: [@kaniini@pleroma.site](https://pleroma.site/kaniini)
+- Features: No Streaming
 
 ### Halcyon
 - Source Code: <https://notabug.org/halcyon-suite/halcyon>
 - Contact: [@halcyon@social.csswg.org](https://social.csswg.org/users/halcyon)
+- Features: Streaming Ready
 
 ### Pinafore
 - Homepage: <https://pinafore.social/>
 - Source Code: <https://github.com/nolanlawson/pinafore>
 - Contact: [@pinafore@mastodon.technology](https://mastodon.technology/users/pinafore)
 - Note: Pleroma support is a secondary goal
+- Features: No Streaming
 
 ### Sengi
 - Source Code: <https://github.com/NicolasConstant/sengi>

lib/pleroma/plugs/http_security_plug.ex

@@ -33,7 +33,22 @@ defp headers do
   end
 
   defp csp_string do
-    protocol = Config.get([Pleroma.Web.Endpoint, :protocol])
+    scheme = Config.get([Pleroma.Web.Endpoint, :url])[:scheme]
+    websocket_url = String.replace(Pleroma.Web.Endpoint.static_url(), "http", "ws")
+
+    connect_src =
+      if Mix.env() == :dev do
+        "connect-src 'self' http://localhost:3035/ " <> websocket_url
+      else
+        "connect-src 'self' " <> websocket_url
+      end
+
+    script_src =
+      if Mix.env() == :dev do
+        "script-src 'self' 'unsafe-eval'"
+      else
+        "script-src 'self'"
+      end
 
     [
       "default-src 'none'",
@@ -43,10 +58,10 @@ defp csp_string do
       "media-src 'self' https:",
       "style-src 'self' 'unsafe-inline'",
       "font-src 'self'",
-      "script-src 'self'",
-      "connect-src 'self' " <> String.replace(Pleroma.Web.Endpoint.static_url(), "http", "ws"),
       "manifest-src 'self'",
-      if protocol == "https" do
+      connect_src,
+      script_src,
+      if scheme == "https" do
         "upgrade-insecure-requests"
       end
     ]

lib/pleroma/web/activity_pub/mrf/keyword_policy.ex

@@ -12,9 +12,9 @@ defp string_matches?(string, pattern) do
     String.match?(string, pattern)
   end
 
-  defp check_reject(%{"object" => %{"content" => content}} = message) do
+  defp check_reject(%{"object" => %{"content" => content, "summary" => summary}} = message) do
     if Enum.any?(Pleroma.Config.get([:mrf_keyword, :reject]), fn pattern ->
-         string_matches?(content, pattern)
+         string_matches?(content, pattern) or string_matches?(summary, pattern)
        end) do
       {:reject, nil}
     else
@@ -22,10 +22,12 @@ defp check_reject(%{"object" => %{"content" => content}} = message) do
     end
   end
 
-  defp check_ftl_removal(%{"to" => to, "object" => %{"content" => content}} = message) do
+  defp check_ftl_removal(
+         %{"to" => to, "object" => %{"content" => content, "summary" => summary}} = message
+       ) do
     if "https://www.w3.org/ns/activitystreams#Public" in to and
          Enum.any?(Pleroma.Config.get([:mrf_keyword, :federated_timeline_removal]), fn pattern ->
-           string_matches?(content, pattern)
+           string_matches?(content, pattern) or string_matches?(summary, pattern)
          end) do
       to = List.delete(to, "https://www.w3.org/ns/activitystreams#Public")
       cc = ["https://www.w3.org/ns/activitystreams#Public" | message["cc"] || []]
@@ -41,14 +43,20 @@ defp check_ftl_removal(%{"to" => to, "object" => %{"content" => content}} = mess
     end
   end
 
-  defp check_replace(%{"object" => %{"content" => content}} = message) do
+  defp check_replace(%{"object" => %{"content" => content, "summary" => summary}} = message) do
-    content =
+    {content, summary} =
-      Enum.reduce(Pleroma.Config.get([:mrf_keyword, :replace]), content, fn {pattern, replacement},
+      Enum.reduce(Pleroma.Config.get([:mrf_keyword, :replace]), {content, summary}, fn {pattern,
-                                                                            acc ->
+                                                                                        replacement},
-        String.replace(acc, pattern, replacement)
+                                                                                       {content_acc,
+                                                                                        summary_acc} ->
+        {String.replace(content_acc, pattern, replacement),
+         String.replace(summary_acc, pattern, replacement)}
       end)
 
-    {:ok, put_in(message["object"]["content"], content)}
+    {:ok,
+     message
+     |> put_in(["object", "content"], content)
+     |> put_in(["object", "summary"], summary)}
   end
 
   @impl true

lib/pleroma/web/rich_media/parser.ex

@@ -9,6 +9,13 @@ defmodule Pleroma.Web.RichMedia.Parser do
     Pleroma.Web.RichMedia.Parsers.OEmbed
   ]
 
+  @hackney_options [
+    pool: :media,
+    timeout: 2_000,
+    recv_timeout: 2_000,
+    max_body: 2_000_000
+  ]
+
   def parse(nil), do: {:error, "No URL provided"}
 
   if Mix.env() == :test do
@@ -28,7 +35,7 @@ def parse(url) do
 
   defp parse_url(url) do
     try do
-      {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: [pool: :media])
+      {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: @hackney_options)
 
       html |> maybe_parse() |> clean_parsed_data() |> check_parsed_data()
     rescue

test/web/activity_pub/mrf/keyword_policy_test.exs

@@ -12,18 +12,35 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicyTest do
   end
 
   describe "rejecting based on keywords" do
-    test "rejects if string matches" do
+    test "rejects if string matches in content" do
       Pleroma.Config.put([:mrf_keyword, :reject], ["pun"])
 
       message = %{
         "type" => "Create",
-        "object" => %{"content" => "just a daily reminder that compLAINer is a good pun"}
+        "object" => %{
+          "content" => "just a daily reminder that compLAINer is a good pun",
+          "summary" => ""
+        }
       }
 
       assert {:reject, nil} == KeywordPolicy.filter(message)
     end
 
-    test "rejects if regex matches" do
+    test "rejects if string matches in summary" do
+      Pleroma.Config.put([:mrf_keyword, :reject], ["pun"])
+
+      message = %{
+        "type" => "Create",
+        "object" => %{
+          "summary" => "just a daily reminder that compLAINer is a good pun",
+          "content" => ""
+        }
+      }
+
+      assert {:reject, nil} == KeywordPolicy.filter(message)
+    end
+
+    test "rejects if regex matches in content" do
       Pleroma.Config.put([:mrf_keyword, :reject], [~r/comp[lL][aA][iI][nN]er/])
 
       assert true ==
@@ -31,7 +48,25 @@ test "rejects if regex matches" do
                  message = %{
                    "type" => "Create",
                    "object" => %{
-                     "content" => "just a daily reminder that #{content} is a good pun"
+                     "content" => "just a daily reminder that #{content} is a good pun",
+                     "summary" => ""
+                   }
+                 }
+
+                 {:reject, nil} == KeywordPolicy.filter(message)
+               end)
+    end
+
+    test "rejects if regex matches in summary" do
+      Pleroma.Config.put([:mrf_keyword, :reject], [~r/comp[lL][aA][iI][nN]er/])
+
+      assert true ==
+               Enum.all?(["complainer", "compLainer", "compLAiNer", "compLAINer"], fn content ->
+                 message = %{
+                   "type" => "Create",
+                   "object" => %{
+                     "summary" => "just a daily reminder that #{content} is a good pun",
+                     "content" => ""
                    }
                  }
 
@@ -41,13 +76,16 @@ test "rejects if regex matches" do
   end
 
   describe "delisting from ftl based on keywords" do
-    test "delists if string matches" do
+    test "delists if string matches in content" do
       Pleroma.Config.put([:mrf_keyword, :federated_timeline_removal], ["pun"])
 
       message = %{
         "to" => ["https://www.w3.org/ns/activitystreams#Public"],
         "type" => "Create",
-        "object" => %{"content" => "just a daily reminder that compLAINer is a good pun"}
+        "object" => %{
+          "content" => "just a daily reminder that compLAINer is a good pun",
+          "summary" => ""
+        }
       }
 
       {:ok, result} = KeywordPolicy.filter(message)
@@ -55,7 +93,24 @@ test "delists if string matches" do
       refute ["https://www.w3.org/ns/activitystreams#Public"] == result["to"]
     end
 
-    test "delists if regex matches" do
+    test "delists if string matches in summary" do
+      Pleroma.Config.put([:mrf_keyword, :federated_timeline_removal], ["pun"])
+
+      message = %{
+        "to" => ["https://www.w3.org/ns/activitystreams#Public"],
+        "type" => "Create",
+        "object" => %{
+          "summary" => "just a daily reminder that compLAINer is a good pun",
+          "content" => ""
+        }
+      }
+
+      {:ok, result} = KeywordPolicy.filter(message)
+      assert ["https://www.w3.org/ns/activitystreams#Public"] == result["cc"]
+      refute ["https://www.w3.org/ns/activitystreams#Public"] == result["to"]
+    end
+
+    test "delists if regex matches in content" do
       Pleroma.Config.put([:mrf_keyword, :federated_timeline_removal], [~r/comp[lL][aA][iI][nN]er/])
 
       assert true ==
@@ -64,7 +119,29 @@ test "delists if regex matches" do
                    "type" => "Create",
                    "to" => ["https://www.w3.org/ns/activitystreams#Public"],
                    "object" => %{
-                     "content" => "just a daily reminder that #{content} is a good pun"
+                     "content" => "just a daily reminder that #{content} is a good pun",
+                     "summary" => ""
+                   }
+                 }
+
+                 {:ok, result} = KeywordPolicy.filter(message)
+
+                 ["https://www.w3.org/ns/activitystreams#Public"] == result["cc"] and
+                   not (["https://www.w3.org/ns/activitystreams#Public"] == result["to"])
+               end)
+    end
+
+    test "delists if regex matches in summary" do
+      Pleroma.Config.put([:mrf_keyword, :federated_timeline_removal], [~r/comp[lL][aA][iI][nN]er/])
+
+      assert true ==
+               Enum.all?(["complainer", "compLainer", "compLAiNer", "compLAINer"], fn content ->
+                 message = %{
+                   "type" => "Create",
+                   "to" => ["https://www.w3.org/ns/activitystreams#Public"],
+                   "object" => %{
+                     "summary" => "just a daily reminder that #{content} is a good pun",
+                     "content" => ""
                    }
                  }
 
@@ -77,20 +154,33 @@ test "delists if regex matches" do
   end
 
   describe "replacing keywords" do
-    test "replaces keyword if string matches" do
+    test "replaces keyword if string matches in content" do
       Pleroma.Config.put([:mrf_keyword, :replace], [{"opensource", "free software"}])
 
       message = %{
         "type" => "Create",
         "to" => ["https://www.w3.org/ns/activitystreams#Public"],
-        "object" => %{"content" => "ZFS is opensource"}
+        "object" => %{"content" => "ZFS is opensource", "summary" => ""}
       }
 
       {:ok, %{"object" => %{"content" => result}}} = KeywordPolicy.filter(message)
       assert result == "ZFS is free software"
     end
 
-    test "replaces keyword if regex matches" do
+    test "replaces keyword if string matches in summary" do
+      Pleroma.Config.put([:mrf_keyword, :replace], [{"opensource", "free software"}])
+
+      message = %{
+        "type" => "Create",
+        "to" => ["https://www.w3.org/ns/activitystreams#Public"],
+        "object" => %{"summary" => "ZFS is opensource", "content" => ""}
+      }
+
+      {:ok, %{"object" => %{"summary" => result}}} = KeywordPolicy.filter(message)
+      assert result == "ZFS is free software"
+    end
+
+    test "replaces keyword if regex matches in content" do
       Pleroma.Config.put([:mrf_keyword, :replace], [
         {~r/open(-|\s)?source\s?(software)?/, "free software"}
       ])
@@ -100,12 +190,30 @@ test "replaces keyword if regex matches" do
                  message = %{
                    "type" => "Create",
                    "to" => ["https://www.w3.org/ns/activitystreams#Public"],
-                   "object" => %{"content" => "ZFS is #{content}"}
+                   "object" => %{"content" => "ZFS is #{content}", "summary" => ""}
                  }
 
                  {:ok, %{"object" => %{"content" => result}}} = KeywordPolicy.filter(message)
                  result == "ZFS is free software"
                end)
     end
+
+    test "replaces keyword if regex matches in summary" do
+      Pleroma.Config.put([:mrf_keyword, :replace], [
+        {~r/open(-|\s)?source\s?(software)?/, "free software"}
+      ])
+
+      assert true ==
+               Enum.all?(["opensource", "open-source", "open source"], fn content ->
+                 message = %{
+                   "type" => "Create",
+                   "to" => ["https://www.w3.org/ns/activitystreams#Public"],
+                   "object" => %{"summary" => "ZFS is #{content}", "content" => ""}
+                 }
+
+                 {:ok, %{"object" => %{"summary" => result}}} = KeywordPolicy.filter(message)
+                 result == "ZFS is free software"
+               end)
+    end
   end
 end