Merge branch 'issue/941' into 'develop'

[#941] add option skip_thread_containment

See merge request pleroma/pleroma!1237

CHANGELOG.md

@@ -50,6 +50,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - MRF: Support for rejecting reports from specific instances (`mrf_simple`)
 - MRF: Support for stripping avatars and banner images from specific instances (`mrf_simple`)
 - MRF: Support for running subchains.
+- Configuration: `skip_thread_containment` option
 
 ### Changed
 - **Breaking:** Configuration: move from Pleroma.Mailer to Pleroma.Emails.Mailer

config/config.exs

@@ -243,7 +243,8 @@
   max_report_comment_size: 1000,
   safe_dm_mentions: false,
   healthcheck: false,
-  remote_post_retention_days: 90
+  remote_post_retention_days: 90,
+  skip_thread_containment: false
 
 config :pleroma, :app_account_creation, enabled: true, max_requests: 25, interval: 1800
 

docs/api/differences_in_mastoapi_responses.md

@@ -82,6 +82,7 @@ Additional parameters can be added to the JSON body/Form data:
 - `show_role` - if true, user's role (e.g admin, moderator) will be exposed to anyone in the API
 - `default_scope` - the scope returned under `privacy` key in Source subentity
 - `pleroma_settings_store` - Opaque user settings to be saved on the backend.
+- `skip_thread_containment` - if true, skip filtering out broken threads
 
 ### Pleroma Settings Store
 Pleroma has mechanism that allows frontends to save blobs of json for each user on the backend. This can be used to save frontend-specific settings for a user that the backend does not need to know about.

docs/config.md

@@ -111,6 +111,7 @@ config :pleroma, Pleroma.Emails.Mailer,
 * `safe_dm_mentions`: If set to true, only mentions at the beginning of a post will be used to address people in direct messages. This is to prevent accidental mentioning of people when talking about them (e.g. "@friend hey i really don't like @enemy"). (Default: `false`)
 * `healthcheck`: if set to true, system data will be shown on ``/api/pleroma/healthcheck``.
 * `remote_post_retention_days`: the default amount of days to retain remote posts when pruning the database
+* `skip_thread_containment`: Skip filter out broken threads. the default is `false`.
 
 ## :app_account_creation
 REST API for creating an account settings

lib/pleroma/user/info.ex

@@ -55,6 +55,8 @@ defmodule Pleroma.User.Info do
       }
     )
 
+    field(:skip_thread_containment, :boolean, default: false)
+
     # Found in the wild
     # ap_id -> Where is this used?
     # bio -> Where is this used?
@@ -220,6 +222,7 @@ def profile_update(info, params) do
       :hide_favorites,
       :background,
       :show_role,
+      :skip_thread_containment,
       :pleroma_settings_store
     ])
   end

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -4,6 +4,7 @@
 
 defmodule Pleroma.Web.ActivityPub.ActivityPub do
   alias Pleroma.Activity
+  alias Pleroma.Config
   alias Pleroma.Conversation
   alias Pleroma.Notification
   alias Pleroma.Object
@@ -73,7 +74,7 @@ defp check_actor_is_active(actor) do
   end
 
   defp check_remote_limit(%{"object" => %{"content" => content}}) when not is_nil(content) do
-    limit = Pleroma.Config.get([:instance, :remote_limit])
+    limit = Config.get([:instance, :remote_limit])
     String.length(content) <= limit
   end
 
@@ -411,8 +412,8 @@ def delete(%Object{data: %{"id" => id, "actor" => actor}} = object, local \\ tru
   end
 
   def block(blocker, blocked, activity_id \\ nil, local \\ true) do
-    outgoing_blocks = Pleroma.Config.get([:activitypub, :outgoing_blocks])
+    outgoing_blocks = Config.get([:activitypub, :outgoing_blocks])
-    unfollow_blocked = Pleroma.Config.get([:activitypub, :unfollow_blocked])
+    unfollow_blocked = Config.get([:activitypub, :unfollow_blocked])
 
     if unfollow_blocked do
       follow_activity = fetch_latest_follow(blocker, blocked)
@@ -557,14 +558,11 @@ defp restrict_visibility(query, %{visibility: visibility})
 
   defp restrict_visibility(query, %{visibility: visibility})
        when visibility in @valid_visibilities do
-    query =
     from(
       a in query,
       where:
         fragment("activity_visibility(?, ?, ?) = ?", a.actor, a.recipients, a.data, ^visibility)
     )
-
-    query
   end
 
   defp restrict_visibility(_query, %{visibility: visibility})
@@ -574,17 +572,24 @@ defp restrict_visibility(_query, %{visibility: visibility})
 
   defp restrict_visibility(query, _visibility), do: query
 
-  defp restrict_thread_visibility(query, %{"user" => %User{ap_id: ap_id}}) do
+  defp restrict_thread_visibility(query, _, %{skip_thread_containment: true} = _),
-    query =
+    do: query
+
+  defp restrict_thread_visibility(
+         query,
+         %{"user" => %User{info: %{skip_thread_containment: true}}},
+         _
+       ),
+       do: query
+
+  defp restrict_thread_visibility(query, %{"user" => %User{ap_id: ap_id}}, _) do
     from(
       a in query,
       where: fragment("thread_visibility(?, (?)->>'id') = true", ^ap_id, a.data)
     )
-
-    query
   end
 
-  defp restrict_thread_visibility(query, _), do: query
+  defp restrict_thread_visibility(query, _, _), do: query
 
   def fetch_user_activities(user, reading_user, params \\ %{}) do
     params =
@@ -863,6 +868,10 @@ defp maybe_order(query, _), do: query
   def fetch_activities_query(recipients, opts \\ %{}) do
     base_query = from(activity in Activity)
 
+    config = %{
+      skip_thread_containment: Config.get([:instance, :skip_thread_containment])
+    }
+
     base_query
     |> maybe_preload_objects(opts)
     |> maybe_preload_bookmarks(opts)
@@ -882,7 +891,7 @@ def fetch_activities_query(recipients, opts \\ %{}) do
     |> restrict_muted(opts)
     |> restrict_media(opts)
     |> restrict_visibility(opts)
-    |> restrict_thread_visibility(opts)
+    |> restrict_thread_visibility(opts, config)
     |> restrict_replies(opts)
     |> restrict_reblogs(opts)
     |> restrict_pinned(opts)

lib/pleroma/web/mastodon_api/mastodon_api_controller.ex

@@ -117,7 +117,15 @@ def update_credentials(%{assigns: %{user: user}} = conn, params) do
       |> Enum.dedup()
 
     info_params =
-      [:no_rich_text, :locked, :hide_followers, :hide_follows, :hide_favorites, :show_role]
+      [
+        :no_rich_text,
+        :locked,
+        :hide_followers,
+        :hide_follows,
+        :hide_favorites,
+        :show_role,
+        :skip_thread_containment
+      ]
       |> Enum.reduce(%{}, fn key, acc ->
         add_if_present(acc, params, to_string(key), key, fn value ->
           {:ok, ControllerHelper.truthy_param?(value)}

lib/pleroma/web/mastodon_api/views/account_view.ex

@@ -124,7 +124,8 @@ defp do_render("account.json", %{user: user} = opts) do
         hide_followers: user.info.hide_followers,
         hide_follows: user.info.hide_follows,
         hide_favorites: user.info.hide_favorites,
-        relationship: relationship
+        relationship: relationship,
+        skip_thread_containment: user.info.skip_thread_containment
       }
     }
     |> maybe_put_role(user, opts[:for])

lib/pleroma/web/streamer.ex

@@ -6,6 +6,7 @@ defmodule Pleroma.Web.Streamer do
   use GenServer
   require Logger
   alias Pleroma.Activity
+  alias Pleroma.Config
   alias Pleroma.Conversation.Participation
   alias Pleroma.Notification
   alias Pleroma.Object
@@ -224,11 +225,10 @@ def push_to_socket(topics, topic, %Activity{data: %{"type" => "Announce"}} = ite
         mutes = user.info.mutes || []
         reblog_mutes = user.info.muted_reblogs || []
 
-        parent = Object.normalize(item)
+        with parent when not is_nil(parent) <- Object.normalize(item),
-
+             true <- Enum.all?([blocks, mutes, reblog_mutes], &(item.actor not in &1)),
-        unless is_nil(parent) or item.actor in blocks or item.actor in mutes or
+             true <- Enum.all?([blocks, mutes], &(parent.data["actor"] not in &1)),
-                 item.actor in reblog_mutes or not ActivityPub.contain_activity(item, user) or
+             true <- thread_containment(item, user) do
-                 parent.data["actor"] in blocks or parent.data["actor"] in mutes do
           send(socket.transport_pid, {:text, represent_update(item, user)})
         end
       else
@@ -264,8 +264,8 @@ def push_to_socket(topics, topic, item) do
         blocks = user.info.blocks || []
         mutes = user.info.mutes || []
 
-        unless item.actor in blocks or item.actor in mutes or
+        with true <- Enum.all?([blocks, mutes], &(item.actor not in &1)),
-                 not ActivityPub.contain_activity(item, user) do
+             true <- thread_containment(item, user) do
           send(socket.transport_pid, {:text, represent_update(item, user)})
         end
       else
@@ -279,4 +279,15 @@ defp internal_topic(topic, socket) when topic in ~w[user direct] do
   end
 
   defp internal_topic(topic, _), do: topic
+
+  @spec thread_containment(Activity.t(), User.t()) :: boolean()
+  defp thread_containment(_activity, %User{info: %{skip_thread_containment: true}}), do: true
+
+  defp thread_containment(activity, user) do
+    if Config.get([:instance, :skip_thread_containment]) do
+      true
+    else
+      ActivityPub.contain_activity(activity, user)
+    end
+  end
 end

lib/pleroma/web/twitter_api/twitter_api_controller.ex

@@ -632,7 +632,15 @@ def raw_empty_array(conn, _params) do
 
   defp build_info_cng(user, params) do
     info_params =
-      ["no_rich_text", "locked", "hide_followers", "hide_follows", "hide_favorites", "show_role"]
+      [
+        "no_rich_text",
+        "locked",
+        "hide_followers",
+        "hide_follows",
+        "hide_favorites",
+        "show_role",
+        "skip_thread_containment"
+      ]
       |> Enum.reduce(%{}, fn key, res ->
         if value = params[key] do
           Map.put(res, key, value == "true")

lib/pleroma/web/twitter_api/views/user_view.ex

@@ -118,7 +118,8 @@ defp do_render("user.json", %{user: user = %User{}} = assigns) do
         "pleroma" =>
           %{
             "confirmation_pending" => user_info.confirmation_pending,
-            "tags" => user.tags
+            "tags" => user.tags,
+            "skip_thread_containment" => user.info.skip_thread_containment
           }
           |> maybe_with_activation_status(user, for_user)
           |> with_notification_settings(user, for_user)

test/web/activity_pub/visibilty_test.exs

@@ -1,6 +1,7 @@
 defmodule Pleroma.Web.ActivityPub.VisibilityTest do
   use Pleroma.DataCase
 
+  alias Pleroma.Activity
   alias Pleroma.Web.ActivityPub.Visibility
   alias Pleroma.Web.CommonAPI
   import Pleroma.Factory
@@ -121,4 +122,46 @@ test "get_visibility", %{
   test "get_visibility with directMessage flag" do
     assert Visibility.get_visibility(%{data: %{"directMessage" => true}}) == "direct"
   end
+
+  describe "entire_thread_visible_for_user?/2" do
+    test "returns false if not found activity", %{user: user} do
+      refute Visibility.entire_thread_visible_for_user?(%Activity{}, user)
+    end
+
+    test "returns true if activity hasn't 'Create' type", %{user: user} do
+      activity = insert(:like_activity)
+      assert Visibility.entire_thread_visible_for_user?(activity, user)
+    end
+
+    test "returns false when invalid recipients", %{user: user} do
+      author = insert(:user)
+
+      activity =
+        insert(:note_activity,
+          note:
+            insert(:note,
+              user: author,
+              data: %{"to" => ["test-user"]}
+            )
+        )
+
+      refute Visibility.entire_thread_visible_for_user?(activity, user)
+    end
+
+    test "returns true if user following to author" do
+      author = insert(:user)
+      user = insert(:user, following: [author.ap_id])
+
+      activity =
+        insert(:note_activity,
+          note:
+            insert(:note,
+              user: author,
+              data: %{"to" => [user.ap_id]}
+            )
+        )
+
+      assert Visibility.entire_thread_visible_for_user?(activity, user)
+    end
+  end
 end

test/web/mastodon_api/account_view_test.exs

@@ -67,7 +67,8 @@ test "Represent a user account" do
         hide_favorites: true,
         hide_followers: false,
         hide_follows: false,
-        relationship: %{}
+        relationship: %{},
+        skip_thread_containment: false
       }
     }
 
@@ -132,7 +133,8 @@ test "Represent a Service(bot) account" do
         hide_favorites: true,
         hide_followers: false,
         hide_follows: false,
-        relationship: %{}
+        relationship: %{},
+        skip_thread_containment: false
       }
     }
 
@@ -233,7 +235,8 @@ test "represent an embedded relationship" do
           domain_blocking: false,
           showing_reblogs: true,
           endorsed: false
-        }
+        },
+        skip_thread_containment: false
       }
     }
 

test/web/mastodon_api/mastodon_api_controller_test.exs

@@ -2539,6 +2539,19 @@ test "updates the user's hide_followers status", %{conn: conn} do
       assert user["pleroma"]["hide_followers"] == true
     end
 
+    test "updates the user's skip_thread_containment option", %{conn: conn} do
+      user = insert(:user)
+
+      response =
+        conn
+        |> assign(:user, user)
+        |> patch("/api/v1/accounts/update_credentials", %{skip_thread_containment: "true"})
+        |> json_response(200)
+
+      assert response["pleroma"]["skip_thread_containment"] == true
+      assert refresh_record(user).info.skip_thread_containment
+    end
+
     test "updates the user's hide_follows status", %{conn: conn} do
       user = insert(:user)
 

test/web/streamer_test.exs

@@ -11,6 +11,16 @@ defmodule Pleroma.Web.StreamerTest do
   alias Pleroma.Web.Streamer
   import Pleroma.Factory
 
+  setup do
+    skip_thread_containment = Pleroma.Config.get([:instance, :skip_thread_containment])
+
+    on_exit(fn ->
+      Pleroma.Config.put([:instance, :skip_thread_containment], skip_thread_containment)
+    end)
+
+    :ok
+  end
+
   test "it sends to public" do
     user = insert(:user)
     other_user = insert(:user)
@@ -68,6 +78,74 @@ test "it sends to public" do
     Task.await(task)
   end
 
+  describe "thread_containment" do
+    test "it doesn't send to user if recipients invalid and thread containment is enabled" do
+      Pleroma.Config.put([:instance, :skip_thread_containment], false)
+      author = insert(:user)
+      user = insert(:user, following: [author.ap_id])
+
+      activity =
+        insert(:note_activity,
+          note:
+            insert(:note,
+              user: author,
+              data: %{"to" => ["TEST-FFF"]}
+            )
+        )
+
+      task = Task.async(fn -> refute_receive {:text, _}, 1_000 end)
+      fake_socket = %{transport_pid: task.pid, assigns: %{user: user}}
+      topics = %{"public" => [fake_socket]}
+      Streamer.push_to_socket(topics, "public", activity)
+
+      Task.await(task)
+    end
+
+    test "it sends message if recipients invalid and thread containment is disabled" do
+      Pleroma.Config.put([:instance, :skip_thread_containment], true)
+      author = insert(:user)
+      user = insert(:user, following: [author.ap_id])
+
+      activity =
+        insert(:note_activity,
+          note:
+            insert(:note,
+              user: author,
+              data: %{"to" => ["TEST-FFF"]}
+            )
+        )
+
+      task = Task.async(fn -> assert_receive {:text, _}, 1_000 end)
+      fake_socket = %{transport_pid: task.pid, assigns: %{user: user}}
+      topics = %{"public" => [fake_socket]}
+      Streamer.push_to_socket(topics, "public", activity)
+
+      Task.await(task)
+    end
+
+    test "it sends message if recipients invalid and thread containment is enabled but user's thread containment is disabled" do
+      Pleroma.Config.put([:instance, :skip_thread_containment], false)
+      author = insert(:user)
+      user = insert(:user, following: [author.ap_id], info: %{skip_thread_containment: true})
+
+      activity =
+        insert(:note_activity,
+          note:
+            insert(:note,
+              user: author,
+              data: %{"to" => ["TEST-FFF"]}
+            )
+        )
+
+      task = Task.async(fn -> assert_receive {:text, _}, 1_000 end)
+      fake_socket = %{transport_pid: task.pid, assigns: %{user: user}}
+      topics = %{"public" => [fake_socket]}
+      Streamer.push_to_socket(topics, "public", activity)
+
+      Task.await(task)
+    end
+  end
+
   test "it doesn't send to blocked users" do
     user = insert(:user)
     blocked_user = insert(:user)

test/web/twitter_api/twitter_api_controller_test.exs

@@ -1495,7 +1495,7 @@ test "it sets and un-sets hide_follows", %{conn: conn} do
           "hide_follows" => "false"
         })
 
-      user = Repo.get!(User, user.id)
+      user = refresh_record(user)
       assert user.info.hide_follows == false
       assert json_response(conn, 200) == UserView.render("user.json", %{user: user, for: user})
     end
@@ -1548,6 +1548,29 @@ test "it sets and un-sets show_role", %{conn: conn} do
       assert json_response(conn, 200) == UserView.render("user.json", %{user: user, for: user})
     end
 
+    test "it sets and un-sets skip_thread_containment", %{conn: conn} do
+      user = insert(:user)
+
+      response =
+        conn
+        |> assign(:user, user)
+        |> post("/api/account/update_profile.json", %{"skip_thread_containment" => "true"})
+        |> json_response(200)
+
+      assert response["pleroma"]["skip_thread_containment"] == true
+      user = refresh_record(user)
+      assert user.info.skip_thread_containment
+
+      response =
+        conn
+        |> assign(:user, user)
+        |> post("/api/account/update_profile.json", %{"skip_thread_containment" => "false"})
+        |> json_response(200)
+
+      assert response["pleroma"]["skip_thread_containment"] == false
+      refute refresh_record(user).info.skip_thread_containment
+    end
+
     test "it locks an account", %{conn: conn} do
       user = insert(:user)
 

test/web/twitter_api/views/user_view_test.exs

@@ -99,7 +99,8 @@ test "A user" do
       "fields" => [],
       "pleroma" => %{
         "confirmation_pending" => false,
-        "tags" => []
+        "tags" => [],
+        "skip_thread_containment" => false
       },
       "rights" => %{"admin" => false, "delete_others_notice" => false},
       "role" => "member"
@@ -154,7 +155,8 @@ test "A user for a given other follower", %{user: user} do
       "fields" => [],
       "pleroma" => %{
         "confirmation_pending" => false,
-        "tags" => []
+        "tags" => [],
+        "skip_thread_containment" => false
       },
       "rights" => %{"admin" => false, "delete_others_notice" => false},
       "role" => "member"
@@ -199,7 +201,8 @@ test "A user that follows you", %{user: user} do
       "fields" => [],
       "pleroma" => %{
         "confirmation_pending" => false,
-        "tags" => []
+        "tags" => [],
+        "skip_thread_containment" => false
       },
       "rights" => %{"admin" => false, "delete_others_notice" => false},
       "role" => "member"
@@ -281,7 +284,8 @@ test "A blocked user for the blocker" do
       "fields" => [],
       "pleroma" => %{
         "confirmation_pending" => false,
-        "tags" => []
+        "tags" => [],
+        "skip_thread_containment" => false
       },
       "rights" => %{"admin" => false, "delete_others_notice" => false},
       "role" => "member"