From b777083f3f396a7d8c357ec968f72679befc691c Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Fri, 25 Oct 2019 19:14:18 +0700 Subject: [PATCH 01/54] Add `also_known_as` field to Pleroma.User --- lib/pleroma/user.ex | 10 ++++-- lib/pleroma/web/activity_pub/activity_pub.ex | 3 +- .../web/activity_pub/transmogrifier.ex | 2 +- ...91025081729_add_also_known_as_to_users.exs | 9 ++++++ priv/static/schemas/litepub-0.1.jsonld | 4 +++ .../tesla_mock/admin@mastdon.example.org.json | 9 ++++-- test/web/activity_pub/transmogrifier_test.exs | 31 +++++++++++++++++++ 7 files changed, 61 insertions(+), 7 deletions(-) create mode 100644 priv/repo/migrations/20191025081729_add_also_known_as_to_users.exs diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 7bef6e281..f25345140 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -105,6 +105,7 @@ defmodule Pleroma.User do field(:discoverable, :boolean, default: false) field(:invisible, :boolean, default: false) field(:skip_thread_containment, :boolean, default: false) + field(:also_known_as, {:array, :string}, default: []) field(:notification_settings, :map, default: %{ @@ -324,7 +325,8 @@ def remote_user_creation(params) do :fields, :following_count, :discoverable, - :invisible + :invisible, + :also_known_as ] ) |> validate_required([:name, :ap_id]) @@ -373,7 +375,8 @@ def update_changeset(struct, params \\ %{}) do :fields, :raw_fields, :pleroma_settings_store, - :discoverable + :discoverable, + :also_known_as ] ) |> unique_constraint(:nickname) @@ -413,7 +416,8 @@ def upgrade_changeset(struct, params \\ %{}, remote? \\ false) do :hide_followers, :discoverable, :hide_followers_count, - :hide_follows_count + :hide_follows_count, + :also_known_as ] ) |> unique_constraint(:nickname) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 07dde3537..dc962673c 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1117,7 +1117,8 @@ defp object_to_user_data(data) do name: data["name"], follower_address: data["followers"], following_address: data["following"], - bio: data["summary"] + bio: data["summary"], + also_known_as: Map.get(data, "alsoKnownAs", []) } # nickname can be nil because of virtual actors diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 9b3ee842b..6ada38e1a 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -616,7 +616,7 @@ def handle_incoming( update_data = new_user_data - |> Map.take([:avatar, :banner, :bio, :name]) + |> Map.take([:avatar, :banner, :bio, :name, :also_known_as]) |> Map.put(:fields, fields) |> Map.put(:locked, locked) |> Map.put(:invisible, invisible) diff --git a/priv/repo/migrations/20191025081729_add_also_known_as_to_users.exs b/priv/repo/migrations/20191025081729_add_also_known_as_to_users.exs new file mode 100644 index 000000000..3d9e0a3cf --- /dev/null +++ b/priv/repo/migrations/20191025081729_add_also_known_as_to_users.exs @@ -0,0 +1,9 @@ +defmodule Pleroma.Repo.Migrations.AddAlsoKnownAsToUsers do + use Ecto.Migration + + def change do + alter table(:users) do + add(:also_known_as, {:array, :string}, default: []) + end + end +end diff --git a/priv/static/schemas/litepub-0.1.jsonld b/priv/static/schemas/litepub-0.1.jsonld index c8e69cab5..509df4bb7 100644 --- a/priv/static/schemas/litepub-0.1.jsonld +++ b/priv/static/schemas/litepub-0.1.jsonld @@ -28,6 +28,10 @@ "oauthRegistrationEndpoint": { "@id": "litepub:oauthRegistrationEndpoint", "@type": "@id" + }, + "alsoKnownAs": { + "@id": "as:alsoKnownAs", + "@type": "@id" } } ] diff --git a/test/fixtures/tesla_mock/admin@mastdon.example.org.json b/test/fixtures/tesla_mock/admin@mastdon.example.org.json index 8159dc20a..9fdd6557c 100644 --- a/test/fixtures/tesla_mock/admin@mastdon.example.org.json +++ b/test/fixtures/tesla_mock/admin@mastdon.example.org.json @@ -9,7 +9,11 @@ "inReplyToAtomUri": "ostatus:inReplyToAtomUri", "conversation": "ostatus:conversation", "toot": "http://joinmastodon.org/ns#", - "Emoji": "toot:Emoji" + "Emoji": "toot:Emoji", + "alsoKnownAs": { + "@id": "as:alsoKnownAs", + "@type": "@id" + } }], "id": "http://mastodon.example.org/users/admin", "type": "Person", @@ -50,5 +54,6 @@ "type": "Image", "mediaType": "image/png", "url": "https://cdn.niu.moe/accounts/headers/000/033/323/original/850b3448fa5fd477.png" - } + }, + "alsoKnownAs": ["http://example.org/users/foo"] } diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 6f7e1da1f..8fd2f5053 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -592,6 +592,37 @@ test "it works for incoming update activities" do assert user.bio == "

Some bio

" end + test "it works with alsoKnownAs" do + {:ok, %Activity{data: %{"actor" => actor}}} = + "test/fixtures/mastodon-post-activity.json" + |> File.read!() + |> Poison.decode!() + |> Transmogrifier.handle_incoming() + + assert User.get_cached_by_ap_id(actor).also_known_as == ["http://example.org/users/foo"] + + {:ok, _activity} = + "test/fixtures/mastodon-update.json" + |> File.read!() + |> Poison.decode!() + |> Map.put("actor", actor) + |> Map.update!("object", fn object -> + object + |> Map.put("actor", actor) + |> Map.put("id", actor) + |> Map.put("alsoKnownAs", [ + "http://mastodon.example.org/users/foo", + "http://example.org/users/bar" + ]) + end) + |> Transmogrifier.handle_incoming() + + assert User.get_cached_by_ap_id(actor).also_known_as == [ + "http://mastodon.example.org/users/foo", + "http://example.org/users/bar" + ] + end + test "it works with custom profile fields" do {:ok, activity} = "test/fixtures/mastodon-post-activity.json" From 61fc739ab8917ccb5a12d6ab6db6130dc231990b Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 30 Oct 2019 18:21:49 +0700 Subject: [PATCH 02/54] Handle "Move" activity --- lib/pleroma/following_relationship.ex | 24 ++++++++++ lib/pleroma/web/activity_pub/activity_pub.ex | 24 ++++++++++ .../web/activity_pub/transmogrifier.ex | 18 +++++++ lib/pleroma/web/activity_pub/visibility.ex | 1 + lib/pleroma/workers/background_worker.ex | 7 +++ test/web/activity_pub/activity_pub_test.exs | 48 +++++++++++++++++++ test/web/activity_pub/transmogrifier_test.exs | 24 ++++++++++ 7 files changed, 146 insertions(+) diff --git a/lib/pleroma/following_relationship.ex b/lib/pleroma/following_relationship.ex index 2ffac17ee..2f89eb4cf 100644 --- a/lib/pleroma/following_relationship.ex +++ b/lib/pleroma/following_relationship.ex @@ -107,4 +107,28 @@ def following(%User{} = user) do [user.follower_address | following] end end + + def move_following(origin, target) do + following_relationships = + __MODULE__ + |> where(following_id: ^origin.id) + |> preload([:follower]) + |> limit(50) + |> Repo.all() + + case following_relationships do + [] -> + :ok + + following_relationships -> + Enum.each(following_relationships, fn following_relationship -> + Repo.transaction(fn -> + Repo.delete(following_relationship) + User.follow(following_relationship.follower, target) + end) + end) + + move_following(origin, target) + end + end end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 6adadacdc..67c2cb659 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -514,6 +514,30 @@ def flag( end end + def move(%User{} = origin, %User{} = target, local \\ true) do + params = %{ + "type" => "Move", + "actor" => origin.ap_id, + "object" => origin.ap_id, + "target" => target.ap_id + } + + with true <- origin.ap_id in target.also_known_as, + {:ok, activity} <- insert(params, local) do + maybe_federate(activity) + + BackgroundWorker.enqueue("move_following", %{ + "origin_id" => origin.id, + "target_id" => target.id + }) + + {:ok, activity} + else + false -> {:error, "Target account must have the origin in `alsoKnownAs`"} + err -> err + end + end + defp fetch_activities_for_context_query(context, opts) do public = [Pleroma.Constants.as_public()] diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index c23f2dcd0..78ee6192a 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -785,6 +785,24 @@ def handle_incoming( end end + def handle_incoming( + %{ + "type" => "Move", + "actor" => origin_actor, + "object" => origin_actor, + "target" => target_actor + }, + _options + ) do + with %User{} = origin_user <- User.get_cached_by_ap_id(origin_actor), + {:ok, %User{} = target_user} <- User.get_or_fetch_by_ap_id(target_actor), + true <- origin_actor in target_user.also_known_as do + ActivityPub.move(origin_user, target_user, false) + else + _e -> :error + end + end + def handle_incoming(_, _), do: :error @spec get_obj_helper(String.t(), Keyword.t()) :: {:ok, Object.t()} | nil diff --git a/lib/pleroma/web/activity_pub/visibility.ex b/lib/pleroma/web/activity_pub/visibility.ex index cd4097493..e172f6d3f 100644 --- a/lib/pleroma/web/activity_pub/visibility.ex +++ b/lib/pleroma/web/activity_pub/visibility.ex @@ -14,6 +14,7 @@ defmodule Pleroma.Web.ActivityPub.Visibility do @spec is_public?(Object.t() | Activity.t() | map()) :: boolean() def is_public?(%Object{data: %{"type" => "Tombstone"}}), do: false def is_public?(%Object{data: data}), do: is_public?(data) + def is_public?(%Activity{data: %{"type" => "Move"}}), do: true def is_public?(%Activity{data: data}), do: is_public?(data) def is_public?(%{"directMessage" => true}), do: false def is_public?(data), do: Utils.label_in_message?(Pleroma.Constants.as_public(), data) diff --git a/lib/pleroma/workers/background_worker.ex b/lib/pleroma/workers/background_worker.ex index 7ffc8eabe..323a4da1e 100644 --- a/lib/pleroma/workers/background_worker.ex +++ b/lib/pleroma/workers/background_worker.ex @@ -71,4 +71,11 @@ def perform(%{"op" => "fetch_data_for_activity", "activity_id" => activity_id}, activity = Activity.get_by_id(activity_id) Pleroma.Web.RichMedia.Helpers.perform(:fetch, activity) end + + def perform(%{"op" => "move_following", "origin_id" => origin_id, "target_id" => target_id}, _) do + origin = User.get_cached_by_id(origin_id) + target = User.get_cached_by_id(target_id) + + Pleroma.FollowingRelationship.move_following(origin, target) + end end diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 0e5eb0c50..3f79593e5 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -4,6 +4,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do use Pleroma.DataCase + use Oban.Testing, repo: Pleroma.Repo + alias Pleroma.Activity alias Pleroma.Builders.ActivityBuilder alias Pleroma.Object @@ -1420,4 +1422,50 @@ test "detects hidden follows" do assert follow_info.hide_follows == true end end + + describe "Move activity" do + test "create" do + %{ap_id: old_ap_id} = old_user = insert(:user) + %{ap_id: new_ap_id} = new_user = insert(:user, also_known_as: [old_ap_id]) + follower = insert(:user) + + User.follow(follower, old_user) + + assert User.following?(follower, old_user) + + assert {:ok, activity} = ActivityPub.move(old_user, new_user) + + assert %Activity{ + actor: ^old_ap_id, + data: %{ + "actor" => ^old_ap_id, + "object" => ^old_ap_id, + "target" => ^new_ap_id, + "type" => "Move" + }, + local: true + } = activity + + params = %{ + "op" => "move_following", + "origin_id" => old_user.id, + "target_id" => new_user.id + } + + assert_enqueued(worker: Pleroma.Workers.BackgroundWorker, args: params) + + Pleroma.Workers.BackgroundWorker.perform(params, nil) + + refute User.following?(follower, old_user) + assert User.following?(follower, new_user) + end + + test "old user must be in the new user's `also_known_as` list" do + old_user = insert(:user) + new_user = insert(:user) + + assert {:error, "Target account must have the origin in `alsoKnownAs`"} = + ActivityPub.move(old_user, new_user) + end + end end diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index b36adcf9b..76e7ad8ea 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -1181,6 +1181,30 @@ test "it correctly processes messages with non-array cc field" do assert ["https://www.w3.org/ns/activitystreams#Public"] == activity.data["cc"] assert [user.follower_address] == activity.data["to"] end + + test "it accepts Move activities" do + old_user = insert(:user) + new_user = insert(:user) + + message = %{ + "@context" => "https://www.w3.org/ns/activitystreams", + "type" => "Move", + "actor" => old_user.ap_id, + "object" => old_user.ap_id, + "target" => new_user.ap_id + } + + assert :error = Transmogrifier.handle_incoming(message) + + {:ok, _new_user} = User.update_and_set_cache(new_user, %{also_known_as: [old_user.ap_id]}) + + assert {:ok, %Activity{} = activity} = Transmogrifier.handle_incoming(message) + assert activity.actor == old_user.ap_id + assert activity.data["actor"] == old_user.ap_id + assert activity.data["object"] == old_user.ap_id + assert activity.data["target"] == new_user.ap_id + assert activity.data["type"] == "Move" + end end describe "prepare outgoing" do From 4b7c11e3f928befaa13daf142a2405284b0d07e5 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Mon, 4 Nov 2019 20:44:24 +0300 Subject: [PATCH 03/54] excluded invisible actors from gets /api/v1/accounts/:id --- lib/pleroma/user.ex | 31 ++++++++++--------- lib/pleroma/web/activity_pub/relay.ex | 1 - .../controllers/account_controller.ex | 2 +- ...91104133100_set_visible_service_actors.exs | 22 +++++++++++++ test/user_test.exs | 19 ++++++++++++ .../controllers/account_controller_test.exs | 23 ++++++++++++++ 6 files changed, 82 insertions(+), 16 deletions(-) create mode 100644 priv/repo/migrations/20191104133100_set_visible_service_actors.exs diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index f8c2db1e1..abd1dd936 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -131,6 +131,8 @@ def auth_active?(%User{}), do: true def visible_for?(user, for_user \\ nil) + def visible_for?(%User{invisible: true}, _), do: false + def visible_for?(%User{id: user_id}, %User{id: for_id}) when user_id == for_id, do: true def visible_for?(%User{} = user, for_user) do @@ -1314,22 +1316,23 @@ def get_or_fetch_by_ap_id(ap_id) do end end - @doc "Creates an internal service actor by URI if missing. Optionally takes nickname for addressing." + @doc """ + Creates an internal service actor by URI if missing. + Optionally takes nickname for addressing. + """ def get_or_create_service_actor_by_ap_id(uri, nickname \\ nil) do - with %User{} = user <- get_cached_by_ap_id(uri) do - user - else - _ -> - {:ok, user} = - %User{} - |> cast(%{}, [:ap_id, :nickname, :local]) - |> put_change(:ap_id, uri) - |> put_change(:nickname, nickname) - |> put_change(:local, true) - |> put_change(:follower_address, uri <> "/followers") - |> Repo.insert() + with user when is_nil(user) <- get_cached_by_ap_id(uri) do + {:ok, user} = + %User{ + invisible: true, + local: true, + ap_id: uri, + nickname: nickname, + follower_address: uri <> "/followers" + } + |> Repo.insert() - user + user end end diff --git a/lib/pleroma/web/activity_pub/relay.ex b/lib/pleroma/web/activity_pub/relay.ex index fc2619680..99a804568 100644 --- a/lib/pleroma/web/activity_pub/relay.ex +++ b/lib/pleroma/web/activity_pub/relay.ex @@ -14,7 +14,6 @@ def get_actor do relay_ap_id() |> User.get_or_create_service_actor_by_ap_id() - {:ok, actor} = User.set_invisible(actor, true) actor end diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index 73fad519e..8f5b91f04 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -238,7 +238,7 @@ def relationships(%{assigns: %{user: _user}} = conn, _), do: json(conn, []) @doc "GET /api/v1/accounts/:id" def show(%{assigns: %{user: for_user}} = conn, %{"id" => nickname_or_id}) do with %User{} = user <- User.get_cached_by_nickname_or_id(nickname_or_id, for: for_user), - true <- User.auth_active?(user) || user.id == for_user.id || User.superuser?(for_user) do + true <- User.visible_for?(user, for_user) do render(conn, "show.json", user: user, for: for_user) else _e -> render_error(conn, :not_found, "Can't find user") diff --git a/priv/repo/migrations/20191104133100_set_visible_service_actors.exs b/priv/repo/migrations/20191104133100_set_visible_service_actors.exs new file mode 100644 index 000000000..62907093c --- /dev/null +++ b/priv/repo/migrations/20191104133100_set_visible_service_actors.exs @@ -0,0 +1,22 @@ +defmodule Pleroma.Repo.Migrations.SetVisibleServiceActors do + use Ecto.Migration + import Ecto.Query + alias Pleroma.Repo + + def up do + user_nicknames = ["relay", "internal.fetch"] + + from( + u in "users", + where: u.nickname in ^user_nicknames, + update: [ + set: [invisible: true] + ] + ) + |> Repo.update_all([]) + end + + def down do + :ok + end +end diff --git a/test/user_test.exs b/test/user_test.exs index 6b1b24ce5..485106b75 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -25,6 +25,25 @@ defmodule Pleroma.UserTest do clear_config([:instance, :account_activation_required]) + describe "service actors" do + test "returns invisible actor" do + uri = "#{Pleroma.Web.Endpoint.url()}/internal/fetch-test" + followers_uri = "#{uri}/followers" + user = User.get_or_create_service_actor_by_ap_id(uri, "internal.fetch-test") + + assert %User{ + nickname: "internal.fetch-test", + invisible: true, + local: true, + ap_id: ^uri, + follower_address: ^followers_uri + } = user + + user2 = User.get_or_create_service_actor_by_ap_id(uri, "internal.fetch-test") + assert user.id == user2.id + end + end + describe "when tags are nil" do test "tagging a user" do user = insert(:user, %{tags: nil}) diff --git a/test/web/mastodon_api/controllers/account_controller_test.exs b/test/web/mastodon_api/controllers/account_controller_test.exs index 8fc2d9300..585cb8a9e 100644 --- a/test/web/mastodon_api/controllers/account_controller_test.exs +++ b/test/web/mastodon_api/controllers/account_controller_test.exs @@ -8,6 +8,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountControllerTest do alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.ActivityPub.InternalFetchActor alias Pleroma.Web.CommonAPI alias Pleroma.Web.OAuth.Token @@ -118,6 +119,28 @@ test "accounts fetches correct account for nicknames beginning with numbers", %{ refute acc_one == acc_two assert acc_two == acc_three end + + test "returns 404 when user is invisible", %{conn: conn} do + user = insert(:user, %{invisible: true}) + + resp = + conn + |> get("/api/v1/accounts/#{user.nickname}") + |> json_response(404) + + assert %{"error" => "Can't find user"} = resp + end + + test "returns 404 for internal.fetch actor", %{conn: conn} do + %User{nickname: "internal.fetch"} = InternalFetchActor.get_actor() + + resp = + conn + |> get("/api/v1/accounts/internal.fetch") + |> json_response(404) + + assert %{"error" => "Can't find user"} = resp + end end describe "user timelines" do From 62bc0657e7ad78a708c1bb6a82eb98aa1f01e819 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Tue, 5 Nov 2019 08:55:41 +0300 Subject: [PATCH 04/54] excluded invisible users from search results --- lib/pleroma/user/search.ex | 5 +++++ test/user_search_test.exs | 8 ++++++++ 2 files changed, 13 insertions(+) diff --git a/lib/pleroma/user/search.ex b/lib/pleroma/user/search.ex index 09664db76..b1bb9d4da 100644 --- a/lib/pleroma/user/search.ex +++ b/lib/pleroma/user/search.ex @@ -45,6 +45,7 @@ defp search_query(query_string, for_user, following) do for_user |> base_query(following) |> filter_blocked_user(for_user) + |> filter_invisible_users() |> filter_blocked_domains(for_user) |> fts_search(query_string) |> trigram_rank(query_string) @@ -98,6 +99,10 @@ defp trigram_rank(query, query_string) do defp base_query(_user, false), do: User defp base_query(user, true), do: User.get_followers_query(user) + defp filter_invisible_users(query) do + from(q in query, where: q.invisible == false) + end + defp filter_blocked_user(query, %User{blocks: blocks}) when length(blocks) > 0 do from(q in query, where: not (q.ap_id in ^blocks)) diff --git a/test/user_search_test.exs b/test/user_search_test.exs index 721af1e5b..98841dbbd 100644 --- a/test/user_search_test.exs +++ b/test/user_search_test.exs @@ -15,6 +15,14 @@ defmodule Pleroma.UserSearchTest do end describe "User.search" do + test "excluded invisible users from results" do + user = insert(:user, %{nickname: "john t1000"}) + insert(:user, %{invisible: true, nickname: "john t800"}) + + [found_user] = User.search("john") + assert found_user.id == user.id + end + test "accepts limit parameter" do Enum.each(0..4, &insert(:user, %{nickname: "john#{&1}"})) assert length(User.search("john", limit: 3)) == 3 From e52955c961a225618f00f8f0fc0e7dcbf5a51e23 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Tue, 5 Nov 2019 10:50:03 +0300 Subject: [PATCH 05/54] update following_relationship.ex --- lib/pleroma/following_relationship.ex | 2 +- test/following_relationship_test.exs | 47 +++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 test/following_relationship_test.exs diff --git a/lib/pleroma/following_relationship.ex b/lib/pleroma/following_relationship.ex index 2ffac17ee..3aff9fb76 100644 --- a/lib/pleroma/following_relationship.ex +++ b/lib/pleroma/following_relationship.ex @@ -101,7 +101,7 @@ def following(%User{} = user) do |> select([r, u], u.follower_address) |> Repo.all() - if not user.local or user.nickname in [nil, "internal.fetch"] do + if not user.local or user.invisible do following else [user.follower_address | following] diff --git a/test/following_relationship_test.exs b/test/following_relationship_test.exs new file mode 100644 index 000000000..93c079814 --- /dev/null +++ b/test/following_relationship_test.exs @@ -0,0 +1,47 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.FollowingRelationshipTest do + use Pleroma.DataCase + + alias Pleroma.FollowingRelationship + alias Pleroma.Web.ActivityPub.InternalFetchActor + alias Pleroma.Web.ActivityPub.Relay + + import Pleroma.Factory + + describe "following/1" do + test "returns following addresses without internal.fetch" do + user = insert(:user) + fetch_actor = InternalFetchActor.get_actor() + FollowingRelationship.follow(fetch_actor, user, "accept") + assert FollowingRelationship.following(fetch_actor) == [user.follower_address] + end + + test "returns following addresses without relay" do + user = insert(:user) + relay_actor = Relay.get_actor() + FollowingRelationship.follow(relay_actor, user, "accept") + assert FollowingRelationship.following(relay_actor) == [user.follower_address] + end + + test "returns following addresses without remote user" do + user = insert(:user) + actor = insert(:user, local: false) + FollowingRelationship.follow(actor, user, "accept") + assert FollowingRelationship.following(actor) == [user.follower_address] + end + + test "returns following addresses with local user" do + user = insert(:user) + actor = insert(:user, local: true) + FollowingRelationship.follow(actor, user, "accept") + + assert FollowingRelationship.following(actor) == [ + actor.follower_address, + user.follower_address + ] + end + end +end From e6d7e27bd603806e96dfc2774f90cadb3cf73a8c Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 12 Nov 2019 18:36:50 +0700 Subject: [PATCH 06/54] Add `allow_following_move` setting to User --- docs/API/differences_in_mastoapi_responses.md | 2 ++ lib/pleroma/following_relationship.ex | 34 ++++++++----------- lib/pleroma/user.ex | 3 ++ .../controllers/account_controller.ex | 1 + .../web/mastodon_api/views/account_view.ex | 7 ++++ ...91025081729_add_also_known_as_to_users.exs | 9 ----- ...191025081729_add_move_support_to_users.exs | 10 ++++++ test/web/activity_pub/activity_pub_test.exs | 6 ++++ .../update_credentials_test.exs | 15 ++++++++ .../mastodon_api/views/account_view_test.exs | 2 +- 10 files changed, 59 insertions(+), 30 deletions(-) delete mode 100644 priv/repo/migrations/20191025081729_add_also_known_as_to_users.exs create mode 100644 priv/repo/migrations/20191025081729_add_move_support_to_users.exs diff --git a/docs/API/differences_in_mastoapi_responses.md b/docs/API/differences_in_mastoapi_responses.md index aca0f5e0e..41965f872 100644 --- a/docs/API/differences_in_mastoapi_responses.md +++ b/docs/API/differences_in_mastoapi_responses.md @@ -57,6 +57,7 @@ Has these additional fields under the `pleroma` object: - `settings_store`: A generic map of settings for frontends. Opaque to the backend. Only returned in `verify_credentials` and `update_credentials` - `chat_token`: The token needed for Pleroma chat. Only returned in `verify_credentials` - `deactivated`: boolean, true when the user is deactivated +- `allow_following_move`: boolean, true when the user allows automatically follow moved following accounts - `unread_conversation_count`: The count of unread conversations. Only returned to the account owner. ### Source @@ -130,6 +131,7 @@ Additional parameters can be added to the JSON body/Form data: - `default_scope` - the scope returned under `privacy` key in Source subentity - `pleroma_settings_store` - Opaque user settings to be saved on the backend. - `skip_thread_containment` - if true, skip filtering out broken threads +- `allow_following_move` - if true, allows automatically follow moved following accounts - `pleroma_background_image` - sets the background image of the user. ### Pleroma Settings Store diff --git a/lib/pleroma/following_relationship.ex b/lib/pleroma/following_relationship.ex index 2f89eb4cf..40538f7bf 100644 --- a/lib/pleroma/following_relationship.ex +++ b/lib/pleroma/following_relationship.ex @@ -109,26 +109,20 @@ def following(%User{} = user) do end def move_following(origin, target) do - following_relationships = - __MODULE__ - |> where(following_id: ^origin.id) - |> preload([:follower]) - |> limit(50) - |> Repo.all() - - case following_relationships do - [] -> - :ok - - following_relationships -> - Enum.each(following_relationships, fn following_relationship -> - Repo.transaction(fn -> - Repo.delete(following_relationship) - User.follow(following_relationship.follower, target) - end) - end) - - move_following(origin, target) + __MODULE__ + |> join(:inner, [r], f in assoc(r, :follower)) + |> where(following_id: ^origin.id) + |> where([r, f], f.allow_following_move == true) + |> limit(50) + |> preload([:follower]) + |> Repo.all() + |> Enum.map(fn following_relationship -> + Repo.delete(following_relationship) + Pleroma.Web.CommonAPI.follow(following_relationship.follower, target) + end) + |> case do + [] -> :ok + _ -> move_following(origin, target) end end end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 8715b37de..d40f6ed08 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -104,6 +104,7 @@ defmodule Pleroma.User do field(:raw_fields, {:array, :map}, default: []) field(:discoverable, :boolean, default: false) field(:invisible, :boolean, default: false) + field(:allow_following_move, :boolean, default: true) field(:skip_thread_containment, :boolean, default: false) field(:also_known_as, {:array, :string}, default: []) @@ -314,6 +315,7 @@ def update_changeset(struct, params \\ %{}) do :hide_followers_count, :hide_follows_count, :hide_favorites, + :allow_following_move, :background, :show_role, :skip_thread_containment, @@ -359,6 +361,7 @@ def upgrade_changeset(struct, params \\ %{}, remote? \\ false) do :hide_follows, :fields, :hide_followers, + :allow_following_move, :discoverable, :hide_followers_count, :hide_follows_count, diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index 73fad519e..7df7dc097 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -152,6 +152,7 @@ def update_credentials(%{assigns: %{user: original_user}} = conn, params) do :hide_favorites, :show_role, :skip_thread_containment, + :allow_following_move, :discoverable ] |> Enum.reduce(%{}, fn key, acc -> diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index e30fed610..7aae7d188 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -163,6 +163,7 @@ defp do_render("show.json", %{user: user} = opts) do |> maybe_put_chat_token(user, opts[:for], opts) |> maybe_put_activation_status(user, opts[:for]) |> maybe_put_follow_requests_count(user, opts[:for]) + |> maybe_put_allow_following_move(user, opts[:for]) |> maybe_put_unread_conversation_count(user, opts[:for]) end @@ -239,6 +240,12 @@ defp maybe_put_notification_settings(data, %User{id: user_id} = user, %User{id: defp maybe_put_notification_settings(data, _, _), do: data + defp maybe_put_allow_following_move(data, %User{id: user_id} = user, %User{id: user_id}) do + Kernel.put_in(data, [:pleroma, :allow_following_move], user.allow_following_move) + end + + defp maybe_put_allow_following_move(data, _, _), do: data + defp maybe_put_activation_status(data, user, %User{is_admin: true}) do Kernel.put_in(data, [:pleroma, :deactivated], user.deactivated) end diff --git a/priv/repo/migrations/20191025081729_add_also_known_as_to_users.exs b/priv/repo/migrations/20191025081729_add_also_known_as_to_users.exs deleted file mode 100644 index 3d9e0a3cf..000000000 --- a/priv/repo/migrations/20191025081729_add_also_known_as_to_users.exs +++ /dev/null @@ -1,9 +0,0 @@ -defmodule Pleroma.Repo.Migrations.AddAlsoKnownAsToUsers do - use Ecto.Migration - - def change do - alter table(:users) do - add(:also_known_as, {:array, :string}, default: []) - end - end -end diff --git a/priv/repo/migrations/20191025081729_add_move_support_to_users.exs b/priv/repo/migrations/20191025081729_add_move_support_to_users.exs new file mode 100644 index 000000000..580b9eb0f --- /dev/null +++ b/priv/repo/migrations/20191025081729_add_move_support_to_users.exs @@ -0,0 +1,10 @@ +defmodule Pleroma.Repo.Migrations.AddMoveSupportToUsers do + use Ecto.Migration + + def change do + alter table(:users) do + add(:also_known_as, {:array, :string}, default: [], null: false) + add(:allow_following_move, :boolean, default: true, null: false) + end + end +end diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 3f79593e5..6c5e5d38e 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -1428,10 +1428,13 @@ test "create" do %{ap_id: old_ap_id} = old_user = insert(:user) %{ap_id: new_ap_id} = new_user = insert(:user, also_known_as: [old_ap_id]) follower = insert(:user) + follower_move_opted_out = insert(:user, allow_following_move: false) User.follow(follower, old_user) + User.follow(follower_move_opted_out, old_user) assert User.following?(follower, old_user) + assert User.following?(follower_move_opted_out, old_user) assert {:ok, activity} = ActivityPub.move(old_user, new_user) @@ -1458,6 +1461,9 @@ test "create" do refute User.following?(follower, old_user) assert User.following?(follower, new_user) + + assert User.following?(follower_move_opted_out, old_user) + refute User.following?(follower_move_opted_out, new_user) end test "old user must be in the new user's `also_known_as` list" do diff --git a/test/web/mastodon_api/controllers/account_controller/update_credentials_test.exs b/test/web/mastodon_api/controllers/account_controller/update_credentials_test.exs index 519b56d6c..77cfce4fa 100644 --- a/test/web/mastodon_api/controllers/account_controller/update_credentials_test.exs +++ b/test/web/mastodon_api/controllers/account_controller/update_credentials_test.exs @@ -103,6 +103,21 @@ test "updates the user's locking status", %{conn: conn} do assert user["locked"] == true end + test "updates the user's allow_following_move", %{conn: conn} do + user = insert(:user) + + assert user.allow_following_move == true + + conn = + conn + |> assign(:user, user) + |> patch("/api/v1/accounts/update_credentials", %{allow_following_move: "false"}) + + assert refresh_record(user).allow_following_move == false + assert user = json_response(conn, 200) + assert user["pleroma"]["allow_following_move"] == false + end + test "updates the user's default scope", %{conn: conn} do user = insert(:user) diff --git a/test/web/mastodon_api/views/account_view_test.exs b/test/web/mastodon_api/views/account_view_test.exs index af88841ed..3aa5d2e3b 100644 --- a/test/web/mastodon_api/views/account_view_test.exs +++ b/test/web/mastodon_api/views/account_view_test.exs @@ -102,7 +102,7 @@ test "Represent the user account for the account owner" do privacy = user.default_scope assert %{ - pleroma: %{notification_settings: ^notification_settings}, + pleroma: %{notification_settings: ^notification_settings, allow_following_move: true}, source: %{privacy: ^privacy} } = AccountView.render("show.json", %{user: user, for: user}) end From 27cd1374e3c1768a1a0b8abb40d5b56e6e021d2d Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 12 Nov 2019 18:48:14 +0700 Subject: [PATCH 07/54] Add a notification for Move activities --- lib/pleroma/notification.ex | 33 ++++++++++----------- lib/pleroma/web/common_api/utils.ex | 21 +++++++++---- test/web/activity_pub/activity_pub_test.exs | 9 ++++++ 3 files changed, 40 insertions(+), 23 deletions(-) diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index b7ecf51e4..f37e7ec67 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -251,10 +251,13 @@ def create_notifications(%Activity{data: %{"to" => _, "type" => "Create"}} = act end end - def create_notifications(%Activity{data: %{"to" => _, "type" => type}} = activity) - when type in ["Like", "Announce", "Follow"] do - users = get_notified_from_activity(activity) - notifications = Enum.map(users, fn user -> create_notification(activity, user) end) + def create_notifications(%Activity{data: %{"type" => type}} = activity) + when type in ["Like", "Announce", "Follow", "Move"] do + notifications = + activity + |> get_notified_from_activity() + |> Enum.map(&create_notification(activity, &1)) + {:ok, notifications} end @@ -276,19 +279,15 @@ def create_notification(%Activity{} = activity, %User{} = user) do def get_notified_from_activity(activity, local_only \\ true) - def get_notified_from_activity( - %Activity{data: %{"to" => _, "type" => type} = _data} = activity, - local_only - ) - when type in ["Create", "Like", "Announce", "Follow"] do - recipients = - [] - |> Utils.maybe_notify_to_recipients(activity) - |> Utils.maybe_notify_mentioned_recipients(activity) - |> Utils.maybe_notify_subscribers(activity) - |> Enum.uniq() - - User.get_users_from_set(recipients, local_only) + def get_notified_from_activity(%Activity{data: %{"type" => type}} = activity, local_only) + when type in ["Create", "Like", "Announce", "Follow", "Move"] do + [] + |> Utils.maybe_notify_to_recipients(activity) + |> Utils.maybe_notify_mentioned_recipients(activity) + |> Utils.maybe_notify_subscribers(activity) + |> Utils.maybe_notify_followers(activity) + |> Enum.uniq() + |> User.get_users_from_set(local_only) end def get_notified_from_activity(_, _local_only), do: [] diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 88a5f434a..43b67d0f0 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -451,6 +451,8 @@ def maybe_notify_to_recipients( recipients ++ to end + def maybe_notify_to_recipients(recipients, _), do: recipients + def maybe_notify_mentioned_recipients( recipients, %Activity{data: %{"to" => _to, "type" => type} = data} = activity @@ -484,12 +486,8 @@ def maybe_notify_subscribers(recipients, %Activity{ recipients end - def maybe_notify_subscribers( - recipients, - %Activity{data: %{"actor" => actor, "type" => type}} = activity - ) - when type == "Create" do - with %User{} = user <- User.get_cached_by_ap_id(actor) do + def maybe_notify_subscribers(recipients, %Activity{data: %{"type" => "Create"}} = activity) do + with %User{} = user <- User.get_cached_by_ap_id(activity.actor) do subscriber_ids = user |> User.subscribers() @@ -502,6 +500,17 @@ def maybe_notify_subscribers( def maybe_notify_subscribers(recipients, _), do: recipients + def maybe_notify_followers(recipients, %Activity{data: %{"type" => "Move"}} = activity) do + with %User{} = user <- User.get_cached_by_ap_id(activity.actor) do + user + |> User.get_followers() + |> Enum.map(& &1.ap_id) + |> Enum.concat(recipients) + end + end + + def maybe_notify_followers(recipients, _), do: recipients + def maybe_extract_mentions(%{"tag" => tag}) do tag |> Enum.filter(fn x -> is_map(x) && x["type"] == "Mention" end) diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 6c5e5d38e..bbae3fcf1 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -8,6 +8,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do alias Pleroma.Activity alias Pleroma.Builders.ActivityBuilder + alias Pleroma.Notification alias Pleroma.Object alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub @@ -1464,6 +1465,14 @@ test "create" do assert User.following?(follower_move_opted_out, old_user) refute User.following?(follower_move_opted_out, new_user) + + activity = %Activity{activity | object: nil} + + assert [%Notification{activity: ^activity}] = + Notification.for_user_since(follower, ~N[2019-04-13 11:22:33]) + + assert [%Notification{activity: ^activity}] = + Notification.for_user_since(follower_move_opted_out, ~N[2019-04-13 11:22:33]) end test "old user must be in the new user's `also_known_as` list" do From 63c493cd536155b98063be5cb680914413597312 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 12 Nov 2019 18:51:12 +0700 Subject: [PATCH 08/54] Update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 51e5424c6..acfe4ceb1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,6 +51,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Admin API: `POST/DELETE /api/pleroma/admin/users/:nickname/permission_group/:permission_group` are deprecated in favor of: `POST/DELETE /api/pleroma/admin/users/permission_group/:permission_group` (both accept `nicknames` array), `DELETE /api/pleroma/admin/users` (`nickname` query param or `nickname` sent in JSON body) is deprecated in favor of: `DELETE /api/pleroma/admin/users` (`nicknames` query array param or `nicknames` sent in JSON body). - Admin API: Add `GET /api/pleroma/admin/relay` endpoint - lists all followed relays - Pleroma API: `POST /api/v1/pleroma/conversations/read` to mark all conversations as read +- ActivityPub: Support `Move` activities - Mastodon API: Add `/api/v1/markers` for managing timeline read markers ### Changed From 768c1a5de172151beb34e6dda13d4fb05e05ed87 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 12 Nov 2019 19:13:19 +0700 Subject: [PATCH 09/54] Fix tests --- lib/pleroma/web/common_api/utils.ex | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 43b67d0f0..cbb64f8d2 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -486,8 +486,12 @@ def maybe_notify_subscribers(recipients, %Activity{ recipients end - def maybe_notify_subscribers(recipients, %Activity{data: %{"type" => "Create"}} = activity) do - with %User{} = user <- User.get_cached_by_ap_id(activity.actor) do + def maybe_notify_subscribers( + recipients, + %Activity{data: %{"actor" => actor, "type" => type}} = activity + ) + when type == "Create" do + with %User{} = user <- User.get_cached_by_ap_id(actor) do subscriber_ids = user |> User.subscribers() From 30af5da33043192dff626e869f2628ffc709f836 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Thu, 14 Nov 2019 23:44:07 +0900 Subject: [PATCH 10/54] Admin API: list all statuses from a given instance --- CHANGELOG.md | 1 + lib/pleroma/web/activity_pub/activity_pub.ex | 26 +++++++++++++ .../web/admin_api/admin_api_controller.ex | 15 ++++++++ lib/pleroma/web/router.ex | 2 + .../admin_api/admin_api_controller_test.exs | 37 +++++++++++++++++++ 5 files changed, 81 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ec084dbd..66644efc9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,6 +60,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Add `/api/v1/markers` for managing timeline read markers - Mastodon API: Add the `recipients` parameter to `GET /api/v1/conversations` - Configuration: `feed` option for user atom feed. +- Admin API: Add `/api/pleroma/admin/instances/:instance/statuses` - lists all statuses from a given instance ### Fixed diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 65dd251f3..8b5bd83e2 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -708,6 +708,17 @@ def fetch_user_activities(user, reading_user, params \\ %{}) do |> Enum.reverse() end + def fetch_instance_activities(params) do + params = + params + |> Map.put("type", ["Create", "Announce"]) + |> Map.put("instance", params["instance"]) + |> Map.put("whole_db", true) + + fetch_activities([Pleroma.Constants.as_public()], params, :offset) + |> Enum.reverse() + end + defp user_activities_recipients(%{"godmode" => true}) do [] end @@ -935,6 +946,20 @@ defp restrict_muted_reblogs(query, %{"muting_user" => %User{} = user}) do defp restrict_muted_reblogs(query, _), do: query + defp restrict_instance(query, %{"instance" => instance}) do + users = + from( + u in User, + select: u.ap_id, + where: fragment("? LIKE ?", u.nickname, ^"%@#{instance}") + ) + |> Repo.all() + + from(activity in query, where: activity.actor in ^users) + end + + defp restrict_instance(query, _), do: query + defp exclude_poll_votes(query, %{"include_poll_votes" => true}), do: query defp exclude_poll_votes(query, _) do @@ -1015,6 +1040,7 @@ def fetch_activities_query(recipients, opts \\ %{}) do |> restrict_reblogs(opts) |> restrict_pinned(opts) |> restrict_muted_reblogs(opts) + |> restrict_instance(opts) |> Activity.restrict_deactivated_users() |> exclude_poll_votes(opts) |> exclude_visibility(opts) diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 30fc01755..2f43f018b 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -226,6 +226,21 @@ def user_show(conn, %{"nickname" => nickname}) do end end + def list_instance_statuses(conn, %{"instance" => instance} = params) do + {page, page_size} = page_params(params) + + activities = + ActivityPub.fetch_instance_activities(%{ + "instance" => instance, + "limit" => page_size, + "offset" => (page - 1) * page_size + }) + + conn + |> put_view(StatusView) + |> render("index.json", %{activities: activities, as: :activity}) + end + def list_user_statuses(conn, %{"nickname" => nickname} = params) do godmode = params["godmode"] == "true" || params["godmode"] == true diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 8fb4aec13..1e2982b67 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -177,6 +177,8 @@ defmodule Pleroma.Web.Router do get("/users/:nickname", AdminAPIController, :user_show) get("/users/:nickname/statuses", AdminAPIController, :list_user_statuses) + get("/instances/:instance/statuses", AdminAPIController, :list_instance_statuses) + get("/reports", AdminAPIController, :list_reports) get("/reports/:id", AdminAPIController, :report_show) put("/reports/:id", AdminAPIController, :report_update_state) diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index 2a9e4f5a0..5958e6462 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -2640,6 +2640,43 @@ test "DELETE /relay", %{admin: admin} do "@#{admin.nickname} unfollowed relay: http://mastodon.example.org/users/admin" end end + + describe "instances" do + test "GET /instances/:instance/statuses" do + admin = insert(:user, is_admin: true) + user = insert(:user, local: false, nickname: "archaeme@archae.me") + user2 = insert(:user, local: false, nickname: "test@test.com") + insert_pair(:note_activity, user: user) + insert(:note_activity, user: user2) + + conn = + build_conn() + |> assign(:user, admin) + |> get("/api/pleroma/admin/instances/archae.me/statuses") + + response = json_response(conn, 200) + + assert length(response) == 2 + + conn = + build_conn() + |> assign(:user, admin) + |> get("/api/pleroma/admin/instances/test.com/statuses") + + response = json_response(conn, 200) + + assert length(response) == 1 + + conn = + build_conn() + |> assign(:user, admin) + |> get("/api/pleroma/admin/instances/nonexistent.com/statuses") + + response = json_response(conn, 200) + + assert length(response) == 0 + end + end end # Needed for testing From c506cc48ef230da30d5786285806de904b725981 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Sat, 16 Nov 2019 18:44:48 +0900 Subject: [PATCH 11/54] Admin API: Error when trying to update reports in the "old" format --- CHANGELOG.md | 1 + lib/pleroma/web/activity_pub/utils.ex | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b4ad91b0d..740facf04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -77,6 +77,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Fix private and direct statuses not being filtered out from the public timeline for an authenticated user (`GET /api/v1/timelines/public`) - Mastodon API: Inability to get some local users by nickname in `/api/v1/accounts/:id_or_nickname` +- Admin API: Error when trying to update reports in the "old" format ## [1.1.2] - 2019-10-18 diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index c45662359..01aacbde3 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -903,7 +903,13 @@ def update_report_state(_, _), do: {:error, "Unsupported state"} def strip_report_status_data(activity) do [actor | reported_activities] = activity.data["object"] - stripped_activities = Enum.map(reported_activities, & &1["id"]) + + stripped_activities = + Enum.map(reported_activities, fn + act when is_map(act) -> act["id"] + act when is_binary(act) -> act + end) + new_data = put_in(activity.data, ["object"], [actor | stripped_activities]) {:ok, %{activity | data: new_data}} From 46eb160135cf408fee87bb76fc46acfc51af901d Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Tue, 19 Nov 2019 20:14:02 +0900 Subject: [PATCH 12/54] AdminAPI: Confirm user account, resend confirmation email --- CHANGELOG.md | 1 + docs/API/admin_api.md | 16 +++++ lib/pleroma/moderation_log.ex | 26 +++++++- lib/pleroma/user.ex | 9 +++ .../web/admin_api/admin_api_controller.ex | 41 ++++++++++-- .../web/admin_api/views/account_view.ex | 3 +- lib/pleroma/web/router.ex | 3 + .../admin_api/admin_api_controller_test.exs | 62 +++++++++++++++++++ 8 files changed, 153 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b4ad91b0d..4b33718a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -67,6 +67,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Add the `recipients` parameter to `GET /api/v1/conversations` - Configuration: `feed` option for user atom feed. - Pleroma API: Add Emoji reactions +- Admin API: `PATCH /api/pleroma/users/confirm_email` to confirm email for multiple users, `PATCH /api/pleroma/users/resend_confirmation_email` to resend confirmation email for multiple users ### Fixed diff --git a/docs/API/admin_api.md b/docs/API/admin_api.md index 9d914c9a6..d943b3ee3 100644 --- a/docs/API/admin_api.md +++ b/docs/API/admin_api.md @@ -878,3 +878,19 @@ Compile time settings (need instance reboot): - Authentication: required - Params: None - Response: JSON, "ok" and 200 status + +## `PATCH /api/pleroma/admin/users/confirm_email` + +### Confirm users' emails + +- Params: + - `nicknames` +- Response: Array of user nicknames + +## `PATCH /api/pleroma/admin/users/resend_confirmation_email` + +### Resend confirmation email + +- Params: + - `nicknames` +- Response: Array of user nicknames diff --git a/lib/pleroma/moderation_log.ex b/lib/pleroma/moderation_log.ex index ffa5dc25d..706f089dc 100644 --- a/lib/pleroma/moderation_log.ex +++ b/lib/pleroma/moderation_log.ex @@ -624,7 +624,31 @@ def get_log_entry_message(%ModerationLog{ "subject" => subjects } }) do - "@#{actor_nickname} force password reset for users: #{users_to_nicknames_string(subjects)}" + "@#{actor_nickname} forced password reset for users: #{users_to_nicknames_string(subjects)}" + end + + @spec get_log_entry_message(ModerationLog) :: String.t() + def get_log_entry_message(%ModerationLog{ + data: %{ + "actor" => %{"nickname" => actor_nickname}, + "action" => "confirm_email", + "subject" => subjects + } + }) do + "@#{actor_nickname} confirmed email for users: #{users_to_nicknames_string(subjects)}" + end + + @spec get_log_entry_message(ModerationLog) :: String.t() + def get_log_entry_message(%ModerationLog{ + data: %{ + "actor" => %{"nickname" => actor_nickname}, + "action" => "resend_confirmation_email", + "subject" => subjects + } + }) do + "@#{actor_nickname} re-sent confirmation email for users: #{ + users_to_nicknames_string(subjects) + }" end defp nicknames_to_string(nicknames) do diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index f8c2db1e1..f031d2179 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -489,6 +489,10 @@ def try_send_confirmation_email(%User{} = user) do end end + def try_send_confirmation_email(users) do + Enum.map(users, &try_send_confirmation_email/1) + end + def needs_update?(%User{local: true}), do: false def needs_update?(%User{local: false, last_refreshed_at: nil}), do: true @@ -1572,6 +1576,11 @@ def toggle_confirmation(%User{} = user) do |> update_and_set_cache() end + @spec toggle_confirmation([User.t()]) :: [{:ok, User.t()} | {:error, Changeset.t()}] + def toggle_confirmation(users) do + Enum.map(users, &toggle_confirmation/1) + end + def get_mascot(%{mascot: %{} = mascot}) when not is_nil(mascot) do mascot end diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 8c1318d1b..c46ce76da 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -335,7 +335,7 @@ def list_users(conn, params) do } with {:ok, users, count} <- Search.user(Map.merge(search_params, filters)), - {:ok, users, count} <- filter_relay_user(users, count), + {:ok, users, count} <- filter_service_users(users, count), do: conn |> json( @@ -347,15 +347,16 @@ def list_users(conn, params) do ) end - defp filter_relay_user(users, count) do - filtered_users = Enum.reject(users, &relay_user?/1) - count = if Enum.any?(users, &relay_user?/1), do: length(filtered_users), else: count + defp filter_service_users(users, count) do + filtered_users = Enum.reject(users, &service_user?/1) + count = if Enum.any?(users, &service_user?/1), do: length(filtered_users), else: count {:ok, filtered_users, count} end - defp relay_user?(user) do - user.ap_id == Relay.relay_ap_id() + defp service_user?(user) do + String.match?(user.ap_id, ~r/.*\/relay$/) or + String.match?(user.ap_id, ~r/.*\/internal\/fetch$/) end @filters ~w(local external active deactivated is_admin is_moderator) @@ -799,6 +800,34 @@ def reload_emoji(conn, _params) do conn |> json("ok") end + def confirm_email(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do + users = nicknames |> Enum.map(&User.get_cached_by_nickname/1) + + User.toggle_confirmation(users) + + ModerationLog.insert_log(%{ + actor: admin, + subject: users, + action: "confirm_email" + }) + + conn |> json("") + end + + def resend_confirmation_email(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do + users = nicknames |> Enum.map(&User.get_cached_by_nickname/1) + + User.try_send_confirmation_email(users) + + ModerationLog.insert_log(%{ + actor: admin, + subject: users, + action: "resend_confirmation_email" + }) + + conn |> json("") + end + def errors(conn, {:error, :not_found}) do conn |> put_status(:not_found) diff --git a/lib/pleroma/web/admin_api/views/account_view.ex b/lib/pleroma/web/admin_api/views/account_view.ex index 6aa7257ce..d9dba5c51 100644 --- a/lib/pleroma/web/admin_api/views/account_view.ex +++ b/lib/pleroma/web/admin_api/views/account_view.ex @@ -36,7 +36,8 @@ def render("show.json", %{user: user}) do "deactivated" => user.deactivated, "local" => user.local, "roles" => User.roles(user), - "tags" => user.tags || [] + "tags" => user.tags || [], + "confirmation_pending" => user.confirmation_pending } end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 9b8b373b8..b654d00c7 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -177,6 +177,9 @@ defmodule Pleroma.Web.Router do get("/users/:nickname", AdminAPIController, :user_show) get("/users/:nickname/statuses", AdminAPIController, :list_user_statuses) + patch("/users/confirm_email", AdminAPIController, :confirm_email) + patch("/users/resend_confirmation_email", AdminAPIController, :resend_confirmation_email) + get("/reports", AdminAPIController, :list_reports) get("/grouped_reports", AdminAPIController, :list_grouped_reports) get("/reports/:id", AdminAPIController, :report_show) diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index 3a4c4d65c..8fabfd963 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -2839,6 +2839,68 @@ test "DELETE /relay", %{admin: admin} do "@#{admin.nickname} unfollowed relay: http://mastodon.example.org/users/admin" end end + + describe "PATCH /confirm_email" do + setup %{conn: conn} do + admin = insert(:user, is_admin: true) + + %{conn: assign(conn, :user, admin), admin: admin} + end + + test "it confirms emails of two users", %{admin: admin} do + [first_user, second_user] = insert_pair(:user, confirmation_pending: true) + + assert first_user.confirmation_pending == true + assert second_user.confirmation_pending == true + + build_conn() + |> assign(:user, admin) + |> patch("/api/pleroma/admin/users/confirm_email", %{ + nicknames: [ + first_user.nickname, + second_user.nickname + ] + }) + + assert first_user.confirmation_pending == true + assert second_user.confirmation_pending == true + + log_entry = Repo.one(ModerationLog) + + assert ModerationLog.get_log_entry_message(log_entry) == + "@#{admin.nickname} confirmed email for users: @#{first_user.nickname}, @#{ + second_user.nickname + }" + end + end + + describe "PATCH /resend_confirmation_email" do + setup %{conn: conn} do + admin = insert(:user, is_admin: true) + + %{conn: assign(conn, :user, admin), admin: admin} + end + + test "it resend emails for two users", %{admin: admin} do + [first_user, second_user] = insert_pair(:user, confirmation_pending: true) + + build_conn() + |> assign(:user, admin) + |> patch("/api/pleroma/admin/users/resend_confirmation_email", %{ + nicknames: [ + first_user.nickname, + second_user.nickname + ] + }) + + log_entry = Repo.one(ModerationLog) + + assert ModerationLog.get_log_entry_message(log_entry) == + "@#{admin.nickname} re-sent confirmation email for users: @#{first_user.nickname}, @#{ + second_user.nickname + }" + end + end end # Needed for testing From b88dbc17c8e404d0dcc16746d4a55210d7c8fd97 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Tue, 19 Nov 2019 22:50:24 +0900 Subject: [PATCH 13/54] Fix tests --- .../admin_api/admin_api_controller_test.exs | 66 ++++++++++++------- 1 file changed, 44 insertions(+), 22 deletions(-) diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index 8fabfd963..9b4f359dc 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -225,7 +225,8 @@ test "Show", %{conn: conn} do "roles" => %{"admin" => false, "moderator" => false}, "tags" => [], "avatar" => User.avatar_url(user) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(user.name || user.nickname) + "display_name" => HTML.strip_tags(user.name || user.nickname), + "confirmation_pending" => false } assert expected == json_response(conn, 200) @@ -634,7 +635,8 @@ test "renders users array for the first page", %{conn: conn, admin: admin} do "local" => true, "tags" => [], "avatar" => User.avatar_url(admin) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(admin.name || admin.nickname) + "display_name" => HTML.strip_tags(admin.name || admin.nickname), + "confirmation_pending" => false }, %{ "deactivated" => user.deactivated, @@ -644,7 +646,8 @@ test "renders users array for the first page", %{conn: conn, admin: admin} do "local" => false, "tags" => ["foo", "bar"], "avatar" => User.avatar_url(user) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(user.name || user.nickname) + "display_name" => HTML.strip_tags(user.name || user.nickname), + "confirmation_pending" => false } ] |> Enum.sort_by(& &1["nickname"]) @@ -685,7 +688,8 @@ test "regular search", %{conn: conn} do "local" => true, "tags" => [], "avatar" => User.avatar_url(user) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(user.name || user.nickname) + "display_name" => HTML.strip_tags(user.name || user.nickname), + "confirmation_pending" => false } ] } @@ -709,7 +713,8 @@ test "search by domain", %{conn: conn} do "local" => true, "tags" => [], "avatar" => User.avatar_url(user) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(user.name || user.nickname) + "display_name" => HTML.strip_tags(user.name || user.nickname), + "confirmation_pending" => false } ] } @@ -733,7 +738,8 @@ test "search by full nickname", %{conn: conn} do "local" => true, "tags" => [], "avatar" => User.avatar_url(user) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(user.name || user.nickname) + "display_name" => HTML.strip_tags(user.name || user.nickname), + "confirmation_pending" => false } ] } @@ -757,7 +763,8 @@ test "search by display name", %{conn: conn} do "local" => true, "tags" => [], "avatar" => User.avatar_url(user) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(user.name || user.nickname) + "display_name" => HTML.strip_tags(user.name || user.nickname), + "confirmation_pending" => false } ] } @@ -781,7 +788,8 @@ test "search by email", %{conn: conn} do "local" => true, "tags" => [], "avatar" => User.avatar_url(user) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(user.name || user.nickname) + "display_name" => HTML.strip_tags(user.name || user.nickname), + "confirmation_pending" => false } ] } @@ -805,7 +813,8 @@ test "regular search with page size", %{conn: conn} do "local" => true, "tags" => [], "avatar" => User.avatar_url(user) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(user.name || user.nickname) + "display_name" => HTML.strip_tags(user.name || user.nickname), + "confirmation_pending" => false } ] } @@ -824,7 +833,8 @@ test "regular search with page size", %{conn: conn} do "local" => true, "tags" => [], "avatar" => User.avatar_url(user2) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(user2.name || user2.nickname) + "display_name" => HTML.strip_tags(user2.name || user2.nickname), + "confirmation_pending" => false } ] } @@ -853,7 +863,8 @@ test "only local users" do "local" => true, "tags" => [], "avatar" => User.avatar_url(user) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(user.name || user.nickname) + "display_name" => HTML.strip_tags(user.name || user.nickname), + "confirmation_pending" => false } ] } @@ -880,7 +891,8 @@ test "only local users with no query", %{admin: old_admin} do "local" => true, "tags" => [], "avatar" => User.avatar_url(user) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(user.name || user.nickname) + "display_name" => HTML.strip_tags(user.name || user.nickname), + "confirmation_pending" => false }, %{ "deactivated" => admin.deactivated, @@ -890,7 +902,8 @@ test "only local users with no query", %{admin: old_admin} do "local" => true, "tags" => [], "avatar" => User.avatar_url(admin) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(admin.name || admin.nickname) + "display_name" => HTML.strip_tags(admin.name || admin.nickname), + "confirmation_pending" => false }, %{ "deactivated" => false, @@ -900,7 +913,8 @@ test "only local users with no query", %{admin: old_admin} do "roles" => %{"admin" => true, "moderator" => false}, "tags" => [], "avatar" => User.avatar_url(old_admin) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(old_admin.name || old_admin.nickname) + "display_name" => HTML.strip_tags(old_admin.name || old_admin.nickname), + "confirmation_pending" => false } ] |> Enum.sort_by(& &1["nickname"]) @@ -929,7 +943,8 @@ test "load only admins", %{conn: conn, admin: admin} do "local" => admin.local, "tags" => [], "avatar" => User.avatar_url(admin) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(admin.name || admin.nickname) + "display_name" => HTML.strip_tags(admin.name || admin.nickname), + "confirmation_pending" => false }, %{ "deactivated" => false, @@ -939,7 +954,8 @@ test "load only admins", %{conn: conn, admin: admin} do "local" => second_admin.local, "tags" => [], "avatar" => User.avatar_url(second_admin) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(second_admin.name || second_admin.nickname) + "display_name" => HTML.strip_tags(second_admin.name || second_admin.nickname), + "confirmation_pending" => false } ] |> Enum.sort_by(& &1["nickname"]) @@ -970,7 +986,8 @@ test "load only moderators", %{conn: conn} do "local" => moderator.local, "tags" => [], "avatar" => User.avatar_url(moderator) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(moderator.name || moderator.nickname) + "display_name" => HTML.strip_tags(moderator.name || moderator.nickname), + "confirmation_pending" => false } ] } @@ -994,7 +1011,8 @@ test "load users with tags list", %{conn: conn} do "local" => user1.local, "tags" => ["first"], "avatar" => User.avatar_url(user1) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(user1.name || user1.nickname) + "display_name" => HTML.strip_tags(user1.name || user1.nickname), + "confirmation_pending" => false }, %{ "deactivated" => false, @@ -1004,7 +1022,8 @@ test "load users with tags list", %{conn: conn} do "local" => user2.local, "tags" => ["second"], "avatar" => User.avatar_url(user2) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(user2.name || user2.nickname) + "display_name" => HTML.strip_tags(user2.name || user2.nickname), + "confirmation_pending" => false } ] |> Enum.sort_by(& &1["nickname"]) @@ -1040,7 +1059,8 @@ test "it works with multiple filters" do "local" => user.local, "tags" => [], "avatar" => User.avatar_url(user) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(user.name || user.nickname) + "display_name" => HTML.strip_tags(user.name || user.nickname), + "confirmation_pending" => false } ] } @@ -1066,7 +1086,8 @@ test "it omits relay user", %{admin: admin} do "local" => true, "tags" => [], "avatar" => User.avatar_url(admin) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(admin.name || admin.nickname) + "display_name" => HTML.strip_tags(admin.name || admin.nickname), + "confirmation_pending" => false } ] } @@ -1135,7 +1156,8 @@ test "PATCH /api/pleroma/admin/users/:nickname/toggle_activation" do "local" => true, "tags" => [], "avatar" => User.avatar_url(user) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(user.name || user.nickname) + "display_name" => HTML.strip_tags(user.name || user.nickname), + "confirmation_pending" => false } log_entry = Repo.one(ModerationLog) From 1fcd579b6d8c26557dcc6f9d3c9f247d03e7b5a4 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Tue, 19 Nov 2019 21:11:15 +0300 Subject: [PATCH 14/54] benchmarks/ added favourites timeline --- benchmarks/load_testing/fetcher.ex | 13 ++++++++++++- benchmarks/load_testing/generator.ex | 18 ++++++++++++++++++ benchmarks/mix/tasks/pleroma/load_testing.ex | 4 ++++ 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/benchmarks/load_testing/fetcher.ex b/benchmarks/load_testing/fetcher.ex index cdc073b2e..776105d34 100644 --- a/benchmarks/load_testing/fetcher.ex +++ b/benchmarks/load_testing/fetcher.ex @@ -57,6 +57,9 @@ def query_timelines(user) do Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities( mastodon_federated_timeline_params ) + end, + "User favourites timeline" => fn -> + Pleroma.Web.ActivityPub.ActivityPub.fetch_favourites(user) end }) @@ -74,6 +77,8 @@ def query_timelines(user) do mastodon_federated_timeline_params ) + user_favourites = Pleroma.Web.ActivityPub.ActivityPub.fetch_favourites(user) + Benchee.run(%{ "Rendering home timeline" => fn -> Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ @@ -95,7 +100,13 @@ def query_timelines(user) do for: user, as: :activity }) - end + end, + "Rendering favorites timeline" => fn -> + Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ + activities: user_favourites, + for: user, + as: :activity}) + end, }) end diff --git a/benchmarks/load_testing/generator.ex b/benchmarks/load_testing/generator.ex index b4432bdb7..f42fd363e 100644 --- a/benchmarks/load_testing/generator.ex +++ b/benchmarks/load_testing/generator.ex @@ -2,6 +2,24 @@ defmodule Pleroma.LoadTesting.Generator do use Pleroma.LoadTesting.Helper alias Pleroma.Web.CommonAPI + def generate_like_activities(user, posts) do + count_likes = Kernel.trunc(length(posts) / 4) + IO.puts("Starting generating #{count_likes} like activities...") + + {time, _} = + :timer.tc(fn -> + Task.async_stream( + Enum.take_random(posts, count_likes), + fn post -> {:ok, _, _} = CommonAPI.favorite(post.id, user) end, + max_concurrency: 10, + timeout: 30_000 + ) + |> Stream.run() + end) + + IO.puts("Inserting like activities take #{to_sec(time)} sec.\n") + end + def generate_users(opts) do IO.puts("Starting generating #{opts[:users_max]} users...") {time, _} = :timer.tc(fn -> do_generate_users(opts) end) diff --git a/benchmarks/mix/tasks/pleroma/load_testing.ex b/benchmarks/mix/tasks/pleroma/load_testing.ex index 4fa3eec49..0a751adac 100644 --- a/benchmarks/mix/tasks/pleroma/load_testing.ex +++ b/benchmarks/mix/tasks/pleroma/load_testing.ex @@ -100,6 +100,10 @@ def run(args) do generate_remote_activities(user, remote_users) + generate_like_activities( + user, Pleroma.Repo.all(Pleroma.Activity.Queries.by_type("Create")) + ) + generate_dms(user, users, opts) {:ok, activity} = generate_long_thread(user, users, opts) From ff9133b978edb637081882e7efd49e2f538e9e15 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 19 Nov 2019 16:01:17 -0600 Subject: [PATCH 15/54] Allow mix tasks to always run with debug logging --- config/config.exs | 1 + config/prod.exs | 2 +- config/test.exs | 2 +- lib/mix/pleroma.ex | 1 + 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/config/config.exs b/config/config.exs index bf2b3f6e2..9ac78c589 100644 --- a/config/config.exs +++ b/config/config.exs @@ -180,6 +180,7 @@ # Configures Elixir's Logger config :logger, :console, + level: :debug, format: "$time $metadata[$level] $message\n", metadata: [:request_id] diff --git a/config/prod.exs b/config/prod.exs index 9c205cbd2..301d2b9cb 100644 --- a/config/prod.exs +++ b/config/prod.exs @@ -20,7 +20,7 @@ config :phoenix, serve_endpoints: true # Do not print debug messages in production -config :logger, level: :warn +config :logger, :console, level: :warn # ## SSL Support # diff --git a/config/test.exs b/config/test.exs index da2778aa7..559ff423e 100644 --- a/config/test.exs +++ b/config/test.exs @@ -15,7 +15,7 @@ method: Pleroma.Captcha.Mock # Print only warnings and errors during test -config :logger, level: :warn +config :logger, :console, level: :warn config :pleroma, :auth, oauth_consumer_strategies: [] diff --git a/lib/mix/pleroma.ex b/lib/mix/pleroma.ex index faeb30e1d..94eeb7d68 100644 --- a/lib/mix/pleroma.ex +++ b/lib/mix/pleroma.ex @@ -6,6 +6,7 @@ defmodule Mix.Pleroma do @doc "Common functions to be reused in mix tasks" def start_pleroma do Application.put_env(:phoenix, :serve_endpoints, false, persistent: true) + Application.put_env(:logger, :console, level: :debug) {:ok, _} = Application.ensure_all_started(:pleroma) end From cb513a599cc3211e6f5e5d1a1710ec2ae85ec37a Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 19 Nov 2019 16:21:30 -0600 Subject: [PATCH 16/54] Allow test builds to have info and debug logs stripped --- config/test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/test.exs b/config/test.exs index 559ff423e..da2778aa7 100644 --- a/config/test.exs +++ b/config/test.exs @@ -15,7 +15,7 @@ method: Pleroma.Captcha.Mock # Print only warnings and errors during test -config :logger, :console, level: :warn +config :logger, level: :warn config :pleroma, :auth, oauth_consumer_strategies: [] From 63711258cebe52c5b0b841a661dc042e8161470c Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 20 Nov 2019 14:25:00 +0700 Subject: [PATCH 17/54] Fix sporadically failing test --- test/web/mastodon_api/controllers/filter_controller_test.exs | 2 +- .../controllers/scheduled_activity_controller_test.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/web/mastodon_api/controllers/filter_controller_test.exs b/test/web/mastodon_api/controllers/filter_controller_test.exs index 5d5b56c8e..550689788 100644 --- a/test/web/mastodon_api/controllers/filter_controller_test.exs +++ b/test/web/mastodon_api/controllers/filter_controller_test.exs @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.MastodonAPI.FilterControllerTest do - use Pleroma.Web.ConnCase, async: true + use Pleroma.Web.ConnCase alias Pleroma.Web.MastodonAPI.FilterView diff --git a/test/web/mastodon_api/controllers/scheduled_activity_controller_test.exs b/test/web/mastodon_api/controllers/scheduled_activity_controller_test.exs index 9ad6a4fa7..ae5fee2bc 100644 --- a/test/web/mastodon_api/controllers/scheduled_activity_controller_test.exs +++ b/test/web/mastodon_api/controllers/scheduled_activity_controller_test.exs @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.MastodonAPI.ScheduledActivityControllerTest do - use Pleroma.Web.ConnCase, async: true + use Pleroma.Web.ConnCase alias Pleroma.Repo alias Pleroma.ScheduledActivity From 1c17f1832983a784156118d1cd488bfde88f092b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 20 Nov 2019 09:14:27 -0600 Subject: [PATCH 18/54] Be specific about format for logs in test as we have tests that check it --- config/test.exs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/config/test.exs b/config/test.exs index da2778aa7..f8188b9e4 100644 --- a/config/test.exs +++ b/config/test.exs @@ -15,7 +15,9 @@ method: Pleroma.Captcha.Mock # Print only warnings and errors during test -config :logger, level: :warn +config :logger, + level: :warn, + format: "[$level] $message" config :pleroma, :auth, oauth_consumer_strategies: [] From c65190c799b6217dca36f9fe4768024b50b725a8 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 20 Nov 2019 11:18:42 -0600 Subject: [PATCH 19/54] Console backend's log format should be prefixed and suffixed with newlines. This is how upstream does it by default. --- config/config.exs | 2 +- config/test.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/config.exs b/config/config.exs index 9ac78c589..1e36d3314 100644 --- a/config/config.exs +++ b/config/config.exs @@ -181,7 +181,7 @@ # Configures Elixir's Logger config :logger, :console, level: :debug, - format: "$time $metadata[$level] $message\n", + format: "\n$time $metadata[$level] $message\n", metadata: [:request_id] config :logger, :ex_syslogger, diff --git a/config/test.exs b/config/test.exs index f8188b9e4..07e70dd4d 100644 --- a/config/test.exs +++ b/config/test.exs @@ -17,7 +17,7 @@ # Print only warnings and errors during test config :logger, level: :warn, - format: "[$level] $message" + format: "\n[$level] $message\n" config :pleroma, :auth, oauth_consumer_strategies: [] From edaae845afc63d558c22019b3b3f87bc510a3bdf Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 20 Nov 2019 11:55:30 -0600 Subject: [PATCH 20/54] Some tasks try to set log level in test environment, but we were stripping logs belog :warn. Ensure console backend logs in test environment are :warn, but mix tasks run in test env do not run with :debug because it's far too verbose. --- config/test.exs | 2 +- lib/mix/pleroma.ex | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/config/test.exs b/config/test.exs index 07e70dd4d..9b737d4d7 100644 --- a/config/test.exs +++ b/config/test.exs @@ -15,7 +15,7 @@ method: Pleroma.Captcha.Mock # Print only warnings and errors during test -config :logger, +config :logger, :console, level: :warn, format: "\n[$level] $message\n" diff --git a/lib/mix/pleroma.ex b/lib/mix/pleroma.ex index 94eeb7d68..73a076a53 100644 --- a/lib/mix/pleroma.ex +++ b/lib/mix/pleroma.ex @@ -6,7 +6,11 @@ defmodule Mix.Pleroma do @doc "Common functions to be reused in mix tasks" def start_pleroma do Application.put_env(:phoenix, :serve_endpoints, false, persistent: true) - Application.put_env(:logger, :console, level: :debug) + + if Pleroma.Config.get(:env) != :test do + Application.put_env(:logger, :console, level: :debug) + end + {:ok, _} = Application.ensure_all_started(:pleroma) end From d38ab4bb89299f643d8529d46e02521a525b84e6 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 21 Nov 2019 16:31:13 +0700 Subject: [PATCH 21/54] Fix user.following_count --- lib/pleroma/user.ex | 56 ++++++++++++++++++++++++--------------------- test/user_test.exs | 5 +++- 2 files changed, 34 insertions(+), 27 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index fcb1d5143..34370be20 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -67,8 +67,7 @@ defmodule Pleroma.User do field(:source_data, :map, default: %{}) field(:note_count, :integer, default: 0) field(:follower_count, :integer, default: 0) - # Should be filled in only for remote users - field(:following_count, :integer, default: nil) + field(:following_count, :integer, default: 0) field(:locked, :boolean, default: false) field(:confirmation_pending, :boolean, default: false) field(:password_reset_pending, :boolean, default: false) @@ -177,19 +176,17 @@ def ap_following(%User{following_address: fa}) when is_binary(fa), do: fa def ap_following(%User{} = user), do: "#{ap_id(user)}/following" def user_info(%User{} = user, args \\ %{}) do - following_count = - Map.get(args, :following_count, user.following_count || following_count(user)) - + following_count = Map.get(args, :following_count, user.following_count) follower_count = Map.get(args, :follower_count, user.follower_count) %{ note_count: user.note_count, locked: user.locked, confirmation_pending: user.confirmation_pending, - default_scope: user.default_scope + default_scope: user.default_scope, + follower_count: follower_count, + following_count: following_count } - |> Map.put(:following_count, following_count) - |> Map.put(:follower_count, follower_count) end def follow_state(%User{} = user, %User{} = target) do @@ -522,14 +519,9 @@ def maybe_direct_follow(%User{} = follower, %User{} = followed) do @doc "A mass follow for local users. Respects blocks in both directions but does not create activities." @spec follow_all(User.t(), list(User.t())) :: {atom(), User.t()} def follow_all(follower, followeds) do - followeds = - Enum.reject(followeds, fn followed -> - blocks?(follower, followed) || blocks?(followed, follower) - end) - - Enum.each(followeds, &follow(follower, &1, "accept")) - - Enum.each(followeds, &update_follower_count/1) + followeds + |> Enum.reject(fn followed -> blocks?(follower, followed) || blocks?(followed, follower) end) + |> Enum.each(&follow(follower, &1, "accept")) set_cache(follower) end @@ -549,11 +541,11 @@ def follow(%User{} = follower, %User{} = followed, state \\ "accept") do true -> FollowingRelationship.follow(follower, followed, state) - follower = maybe_update_following_count(follower) - {:ok, _} = update_follower_count(followed) - set_cache(follower) + follower + |> update_following_count() + |> set_cache() end end @@ -561,11 +553,12 @@ def unfollow(%User{} = follower, %User{} = followed) do if following?(follower, followed) and follower.ap_id != followed.ap_id do FollowingRelationship.unfollow(follower, followed) - follower = maybe_update_following_count(follower) - {:ok, followed} = update_follower_count(followed) - set_cache(follower) + {:ok, follower} = + follower + |> update_following_count() + |> set_cache() {:ok, follower, Utils.fetch_latest_follow(follower, followed)} else @@ -895,8 +888,8 @@ def update_follower_count(%User{} = user) do end end - @spec maybe_update_following_count(User.t()) :: User.t() - def maybe_update_following_count(%User{local: false} = user) do + @spec update_following_count(User.t()) :: User.t() + def update_following_count(%User{local: false} = user) do if Pleroma.Config.get([:instance, :external_user_synchronization]) do maybe_fetch_follow_information(user) else @@ -904,7 +897,13 @@ def maybe_update_following_count(%User{local: false} = user) do end end - def maybe_update_following_count(user), do: user + def update_following_count(%User{local: true} = user) do + following_count = FollowingRelationship.following_count(user) + + user + |> follow_information_changeset(%{following_count: following_count}) + |> Repo.update!() + end def set_unread_conversation_count(%User{local: true} = user) do unread_query = Participation.unread_conversation_count_for_user(user) @@ -1097,7 +1096,12 @@ def deactivate(users, status) when is_list(users) do def deactivate(%User{} = user, status) do with {:ok, user} <- set_activation_status(user, status) do - Enum.each(get_followers(user), &invalidate_cache/1) + user + |> get_followers() + |> Enum.filter(& &1.local) + |> Enum.each(fn follower -> + follower |> update_following_count() |> set_cache() + end) # Only update local user counts, remote will be update during the next pull. user diff --git a/test/user_test.exs b/test/user_test.exs index 8fdb6b25f..8c0d45098 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -148,9 +148,10 @@ test "follow takes a user and another user" do {:ok, user} = User.follow(user, followed) user = User.get_cached_by_id(user.id) - followed = User.get_cached_by_ap_id(followed.ap_id) + assert followed.follower_count == 1 + assert user.following_count == 1 assert User.ap_followers(followed) in User.following(user) end @@ -952,12 +953,14 @@ test "hide a user from friends" do user2 = insert(:user) {:ok, user2} = User.follow(user2, user) + assert user2.following_count == 1 assert User.following_count(user2) == 1 {:ok, _user} = User.deactivate(user) info = User.get_cached_user_info(user2) + assert refresh_record(user2).following_count == 0 assert info.following_count == 0 assert User.following_count(user2) == 0 assert [] = User.get_friends(user2) From 8ffd1c792cbc8cc8bab486119f80a30039c655cb Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 21 Nov 2019 18:22:18 +0700 Subject: [PATCH 22/54] Remove version mismatch warning --- mix.exs | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/mix.exs b/mix.exs index 81ce4f25c..60c7fe3f6 100644 --- a/mix.exs +++ b/mix.exs @@ -194,27 +194,21 @@ defp version(version) do identifier_filter = ~r/[^0-9a-z\-]+/i # Pre-release version, denoted from patch version with a hyphen - {git_tag, git_pre_release} = + git_pre_release = with {tag, 0} <- System.cmd("git", ["describe", "--tags", "--abbrev=0"], stderr_to_stdout: true), - tag = String.trim(tag), - {describe, 0} <- System.cmd("git", ["describe", "--tags", "--abbrev=8"]), - describe = String.trim(describe), - ahead <- String.replace(describe, tag, ""), - ahead <- String.trim_leading(ahead, "-") do - {String.replace_prefix(tag, "v", ""), if(ahead != "", do: String.trim(ahead))} + {describe, 0} <- System.cmd("git", ["describe", "--tags", "--abbrev=8"]) do + describe + |> String.trim() + |> String.replace(String.trim(tag), "") + |> String.trim_leading("-") + |> String.trim() else _ -> {commit_hash, 0} = System.cmd("git", ["rev-parse", "--short", "HEAD"]) - {nil, "0-g" <> String.trim(commit_hash)} + "0-g" <> String.trim(commit_hash) end - if git_tag && version != git_tag do - Mix.shell().error( - "Application version #{inspect(version)} does not match git tag #{inspect(git_tag)}" - ) - end - # Branch name as pre-release version component, denoted with a dot branch_name = with {branch_name, 0} <- System.cmd("git", ["rev-parse", "--abbrev-ref", "HEAD"]), From 320690f7a5bf1622e25114b6ad4c0b0c5a252748 Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 21 Nov 2019 14:03:01 +0100 Subject: [PATCH 23/54] Object: Use inspect to log missing preload. --- lib/pleroma/object.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index d9b41d710..cde0eddd9 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -63,7 +63,7 @@ def get_by_ap_id(ap_id) do end defp warn_on_no_object_preloaded(ap_id) do - "Object.normalize() called without preloaded object (#{ap_id}). Consider preloading the object" + "Object.normalize() called without preloaded object (#{inspect(ap_id)}). Consider preloading the object" |> Logger.debug() Logger.debug("Backtrace: #{inspect(Process.info(:erlang.self(), :current_stacktrace))}") From dc9303eac563d23aebce04ad03e55ab072e548ed Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 21 Nov 2019 15:03:27 +0000 Subject: [PATCH 24/54] Revert "Merge branch 'patch-1' into 'develop'" This reverts merge request !1829 --- config/releases.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/releases.exs b/config/releases.exs index 36c493673..98c5ceccd 100644 --- a/config/releases.exs +++ b/config/releases.exs @@ -1,6 +1,6 @@ import Config -config :pleroma, :instance, static: "/var/lib/pleroma/static" +config :pleroma, :instance, static_dir: "/var/lib/pleroma/static" config :pleroma, Pleroma.Uploaders.Local, uploads: "/var/lib/pleroma/uploads" config_path = System.get_env("PLEROMA_CONFIG_PATH") || "/etc/pleroma/config.exs" From fbc379d68990b166e1034cfff252f975f793eba1 Mon Sep 17 00:00:00 2001 From: Alex Schroeder Date: Sun, 17 Nov 2019 21:58:30 +0100 Subject: [PATCH 25/54] Add CLI list users command --- CHANGELOG.md | 1 + docs/administration/CLI_tasks/user.md | 5 +++++ lib/mix/tasks/pleroma/user.ex | 18 ++++++++++++++++++ 3 files changed, 24 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d9b97f0b..e237ec731 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Authentication: Added rate limit for password-authorized actions / login existence checks - Static Frontend: Add the ability to render user profiles and notices server-side without requiring JS app. - Mix task to re-count statuses for all users (`mix pleroma.count_statuses`) +- Mix task to list all users (`mix pleroma.user list`) - Support for `X-Forwarded-For` and similar HTTP headers which used by reverse proxies to pass a real user IP address to the backend. Must not be enabled unless your instance is behind at least one reverse proxy (such as Nginx, Apache HTTPD or Varnish Cache). - MRF: New module which handles incoming posts based on their age. By default, all incoming posts that are older than 2 days will be unlisted and not shown to their followers.
diff --git a/docs/administration/CLI_tasks/user.md b/docs/administration/CLI_tasks/user.md index cf120f2c9..96b2d9e6a 100644 --- a/docs/administration/CLI_tasks/user.md +++ b/docs/administration/CLI_tasks/user.md @@ -15,6 +15,11 @@ $PREFIX new [] - `--admin`/`--no-admin` - whether the user should be an admin - `-y`, `--assume-yes`/`--no-assume-yes` - whether to assume yes to all questions +## List local users +```sh +$PREFIX list +``` + ## Generate an invite link ```sh $PREFIX invite [] diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex index 4e3b80db3..bc8eacda8 100644 --- a/lib/mix/tasks/pleroma/user.ex +++ b/lib/mix/tasks/pleroma/user.ex @@ -364,6 +364,24 @@ def run(["sign_out", nickname]) do end end + def run(["list"]) do + start_pleroma() + + Pleroma.User.Query.build(%{local: true}) + |> Pleroma.RepoStreamer.chunk_stream(500) + |> Stream.each(fn users -> + users + |> Enum.each(fn user -> + shell_info( + "#{user.nickname} moderator: #{user.info.is_moderator}, admin: #{user.info.is_admin}, locked: #{ + user.info.locked + }, deactivated: #{user.info.deactivated}" + ) + end) + end) + |> Stream.run() + end + defp set_moderator(user, value) do {:ok, user} = user From b64c7b35d429546695550b37ca1e690b46c86f5d Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Fri, 22 Nov 2019 09:11:36 +0900 Subject: [PATCH 26/54] Swap map with each --- lib/pleroma/user.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index f031d2179..f1830509c 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -490,7 +490,7 @@ def try_send_confirmation_email(%User{} = user) do end def try_send_confirmation_email(users) do - Enum.map(users, &try_send_confirmation_email/1) + Enum.each(users, &try_send_confirmation_email/1) end def needs_update?(%User{local: true}), do: false From 1d6970baf1fd09c357740a380351adf44f247279 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Fri, 22 Nov 2019 14:17:12 +0300 Subject: [PATCH 27/54] fix --- benchmarks/load_testing/fetcher.ex | 36 ++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/benchmarks/load_testing/fetcher.ex b/benchmarks/load_testing/fetcher.ex index 776105d34..a45a71d4a 100644 --- a/benchmarks/load_testing/fetcher.ex +++ b/benchmarks/load_testing/fetcher.ex @@ -57,9 +57,6 @@ def query_timelines(user) do Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities( mastodon_federated_timeline_params ) - end, - "User favourites timeline" => fn -> - Pleroma.Web.ActivityPub.ActivityPub.fetch_favourites(user) end }) @@ -77,8 +74,6 @@ def query_timelines(user) do mastodon_federated_timeline_params ) - user_favourites = Pleroma.Web.ActivityPub.ActivityPub.fetch_favourites(user) - Benchee.run(%{ "Rendering home timeline" => fn -> Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ @@ -102,10 +97,33 @@ def query_timelines(user) do }) end, "Rendering favorites timeline" => fn -> - Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ - activities: user_favourites, - for: user, - as: :activity}) + conn = Phoenix.ConnTest.build_conn(:get, "http://localhost:4001/api/v1/favourites", nil) + Pleroma.Web.MastodonAPI.StatusController.favourites( + %Plug.Conn{conn | + assigns: %{user: user}, + query_params: %{"limit" => "0"}, + body_params: %{}, + cookies: %{}, + params: %{}, + path_params: %{}, + private: %{ + Pleroma.Web.Router => {[], %{}}, + phoenix_router: Pleroma.Web.Router, + phoenix_action: :favourites, + phoenix_controller: Pleroma.Web.MastodonAPI.StatusController, + phoenix_endpoint: Pleroma.Web.Endpoint, + phoenix_format: "json", + phoenix_layout: {Pleroma.Web.LayoutView, "app.html"}, + phoenix_recycled: true, + + phoenix_view: Pleroma.Web.MastodonAPI.StatusView, + plug_session: %{"user_id" => user.id}, + plug_session_fetch: :done, + plug_session_info: :write, + plug_skip_csrf_protection: true + } + }, + %{}) end, }) end From 4a5bb7eb021aa828fc24dfb7dbfdb1711e191daa Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 22 Nov 2019 19:58:39 +0300 Subject: [PATCH 28/54] OTP releases: only set name and distribution type if not set already Fixes not being able to run two OTP releases on one machine because of the name conflict. --- rel/env.sh.eex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rel/env.sh.eex b/rel/env.sh.eex index a4ce25295..e1b87102d 100644 --- a/rel/env.sh.eex +++ b/rel/env.sh.eex @@ -8,5 +8,5 @@ # fi # Set the release to work across nodes -export RELEASE_DISTRIBUTION=name -export RELEASE_NODE=<%= @release.name %>@127.0.0.1 +export RELEASE_DISTRIBUTION="${RELEASE_DISTRIBUTION:-name}" +export RELEASE_NODE="${RELEASE_NODE:-<%= @release.name %>@127.0.0.1}" From 1636cc5b7e2ad324c828c993d5fb39ac9cdb40cc Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sat, 23 Nov 2019 14:06:19 +0300 Subject: [PATCH 29/54] Removed users.info and remaining usages. --- benchmarks/load_testing/generator.ex | 1 - lib/pleroma/user.ex | 5 +--- .../20191123103423_remove_info_from_users.exs | 9 +++++++ test/support/factory.ex | 1 - test/user_test.exs | 25 +++---------------- 5 files changed, 14 insertions(+), 27 deletions(-) create mode 100644 priv/repo/migrations/20191123103423_remove_info_from_users.exs diff --git a/benchmarks/load_testing/generator.ex b/benchmarks/load_testing/generator.ex index b4432bdb7..4505d192f 100644 --- a/benchmarks/load_testing/generator.ex +++ b/benchmarks/load_testing/generator.ex @@ -31,7 +31,6 @@ defp generate_user_data(i) do password_hash: "$pbkdf2-sha512$160000$bU.OSFI7H/yqWb5DPEqyjw$uKp/2rmXw12QqnRRTqTtuk2DTwZfF8VR4MYW2xMeIlqPR/UX1nT1CEKVUx2CowFMZ5JON8aDvURrZpJjSgqXrg", bio: "Tester Number #{i}", - info: %{}, local: remote } diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index fcb1d5143..e4656af3d 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -119,8 +119,6 @@ defmodule Pleroma.User do has_many(:registrations, Registration) has_many(:deliveries, Delivery) - field(:info, :map, default: %{}) - timestamps() end @@ -244,7 +242,6 @@ def remote_user_creation(params) do params = params - |> Map.put(:info, params[:info] || %{}) |> truncate_if_exists(:name, name_limit) |> truncate_if_exists(:bio, bio_limit) |> truncate_fields_param() @@ -1226,7 +1223,7 @@ def external_users_query do def external_users(opts \\ []) do query = external_users_query() - |> select([u], struct(u, [:id, :ap_id, :info])) + |> select([u], struct(u, [:id, :ap_id])) query = if opts[:max_id], diff --git a/priv/repo/migrations/20191123103423_remove_info_from_users.exs b/priv/repo/migrations/20191123103423_remove_info_from_users.exs new file mode 100644 index 000000000..b251255ea --- /dev/null +++ b/priv/repo/migrations/20191123103423_remove_info_from_users.exs @@ -0,0 +1,9 @@ +defmodule Pleroma.Repo.Migrations.RemoveInfoFromUsers do + use Ecto.Migration + + def change do + alter table(:users) do + remove(:info, :map, default: %{}) + end + end +end diff --git a/test/support/factory.ex b/test/support/factory.ex index e3f797f64..bb8a64e72 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -31,7 +31,6 @@ def user_factory do nickname: sequence(:nickname, &"nick#{&1}"), password_hash: Comeonin.Pbkdf2.hashpwsalt("test"), bio: sequence(:bio, &"Tester Number #{&1}"), - info: %{}, last_digest_emailed_at: NaiveDateTime.utc_now() } diff --git a/test/user_test.exs b/test/user_test.exs index 8fdb6b25f..e07ddd097 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -347,18 +347,6 @@ test "it sets the password_hash and ap_id" do assert changeset.changes.follower_address == "#{changeset.changes.ap_id}/followers" end - - test "it ensures info is not nil" do - changeset = User.register_changeset(%User{}, @full_user_data) - - assert changeset.valid? - - {:ok, user} = - changeset - |> Repo.insert() - - refute is_nil(user.info) - end end describe "user registration, with :account_activation_required" do @@ -412,8 +400,7 @@ test "gets an existing user by ap_id" do :user, local: false, nickname: "admin@mastodon.example.org", - ap_id: ap_id, - info: %{} + ap_id: ap_id ) {:ok, fetched_user} = User.get_or_fetch(ap_id) @@ -474,8 +461,7 @@ test "updates an existing user, if stale" do local: false, nickname: "admin@mastodon.example.org", ap_id: "http://mastodon.example.org/users/admin", - last_refreshed_at: a_week_ago, - info: %{} + last_refreshed_at: a_week_ago ) assert orig_user.last_refreshed_at == a_week_ago @@ -516,7 +502,6 @@ test "returns an ap_followers link for a user" do name: "Someone", nickname: "a@b.de", ap_id: "http...", - info: %{some: "info"}, avatar: %{some: "avatar"} } @@ -1121,8 +1106,7 @@ test "with an overly long bio" do ap_id: user.ap_id, name: user.name, nickname: user.nickname, - bio: String.duplicate("h", current_max_length + 1), - info: %{} + bio: String.duplicate("h", current_max_length + 1) } assert {:ok, %User{}} = User.insert_or_update_user(data) @@ -1135,8 +1119,7 @@ test "with an overly long display name" do data = %{ ap_id: user.ap_id, name: String.duplicate("h", current_max_length + 1), - nickname: user.nickname, - info: %{} + nickname: user.nickname } assert {:ok, %User{}} = User.insert_or_update_user(data) From 9f2993044098ced1585c8886c16da19dd046b5fd Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sat, 23 Nov 2019 22:55:41 +0300 Subject: [PATCH 30/54] fetcher: move local object checking into a reusable function --- lib/pleroma/object.ex | 4 ++++ lib/pleroma/object/fetcher.ex | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index cde0eddd9..b4ed3a9b2 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -255,4 +255,8 @@ def update_data(%Object{data: data} = object, attrs \\ %{}) do |> Object.change(%{data: Map.merge(data || %{}, attrs)}) |> Repo.update() end + + def local?(%Object{data: %{"id" => id}}) do + String.starts_with?(id, Pleroma.Web.base_url() <> "/") + end end diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 9a9a46550..4d71c91a8 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -49,7 +49,7 @@ defp reinject_object(struct, data) do end def refetch_object(%Object{data: %{"id" => id}} = object) do - with {:local, false} <- {:local, String.starts_with?(id, Pleroma.Web.base_url() <> "/")}, + with {:local, false} <- {:local, Object.local?(object)}, {:ok, data} <- fetch_and_contain_remote_object_from_id(id), {:ok, object} <- reinject_object(object, data) do {:ok, object} From d3656c2725fa80ea66dd9de2d74893fe9888ef57 Mon Sep 17 00:00:00 2001 From: Sadposter Date: Mon, 25 Nov 2019 09:53:11 +0000 Subject: [PATCH 31/54] add ability to set a custom user-agent string --- config/config.exs | 1 + docs/configuration/cheatsheet.md | 7 +++++++ lib/pleroma/application.ex | 10 ++++++++-- test/http/request_builder_test.exs | 10 ++++++++++ 4 files changed, 26 insertions(+), 2 deletions(-) diff --git a/config/config.exs b/config/config.exs index 1e36d3314..b60ffef7d 100644 --- a/config/config.exs +++ b/config/config.exs @@ -209,6 +209,7 @@ config :pleroma, :http, proxy_url: nil, send_user_agent: true, + user_agent: :default, adapter: [ ssl_options: [ # Workaround for remote server certificate chain issues diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 07d9a1d45..082906293 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -74,6 +74,13 @@ You shouldn't edit the base config directly to avoid breakages and merge conflic * `dynamic_configuration`: Allow transferring configuration to DB with the subsequent customization from Admin api. +## :http + +* `proxy_url`: an upstream proxy to fetch posts and/or media with, (default: `nil`) +* `send_user_agent`: should we include a user agent with HTTP requests? (default: `true`) +* `user_agent`: what user agent should we use? (default: `:default`), must be string or `:default` +* `adapter`: array of hackney options + ## Federation ### MRF policies diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 2b6a55f98..9dbd1e26b 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -17,8 +17,14 @@ def named_version, do: @name <> " " <> @version def repository, do: @repository def user_agent do - info = "#{Pleroma.Web.base_url()} <#{Pleroma.Config.get([:instance, :email], "")}>" - named_version() <> "; " <> info + case Pleroma.Config.get([:http, :user_agent], :default) do + :default -> + info = "#{Pleroma.Web.base_url()} <#{Pleroma.Config.get([:instance, :email], "")}>" + named_version() <> "; " <> info + + custom -> + custom + end end # See http://elixir-lang.org/docs/stable/elixir/Application.html diff --git a/test/http/request_builder_test.exs b/test/http/request_builder_test.exs index 170ca916f..80ef25d7b 100644 --- a/test/http/request_builder_test.exs +++ b/test/http/request_builder_test.exs @@ -16,11 +16,21 @@ test "don't send pleroma user agent" do test "send pleroma user agent" do Pleroma.Config.put([:http, :send_user_agent], true) + Pleroma.Config.put([:http, :user_agent], :default) assert RequestBuilder.headers(%{}, []) == %{ headers: [{"User-Agent", Pleroma.Application.user_agent()}] } end + + test "send custom user agent" do + Pleroma.Config.put([:http, :send_user_agent], true) + Pleroma.Config.put([:http, :user_agent], "totally-not-pleroma") + + assert RequestBuilder.headers(%{}, []) == %{ + headers: [{"User-Agent", "totally-not-pleroma"}] + } + end end describe "add_optional_params/3" do From e2dc89069d6d77a10848e53f321e2e464f539a25 Mon Sep 17 00:00:00 2001 From: Sadposter Date: Mon, 25 Nov 2019 11:26:42 +0000 Subject: [PATCH 32/54] move documentation section for http client --- docs/configuration/cheatsheet.md | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 082906293..dc2f55229 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -74,13 +74,6 @@ You shouldn't edit the base config directly to avoid breakages and merge conflic * `dynamic_configuration`: Allow transferring configuration to DB with the subsequent customization from Admin api. -## :http - -* `proxy_url`: an upstream proxy to fetch posts and/or media with, (default: `nil`) -* `send_user_agent`: should we include a user agent with HTTP requests? (default: `true`) -* `user_agent`: what user agent should we use? (default: `:default`), must be string or `:default` -* `adapter`: array of hackney options - ## Federation ### MRF policies @@ -355,7 +348,17 @@ Available caches: * `:activity_pub` - activity pub routes (except question activities). Defaults to `nil` (no expiration). * `:activity_pub_question` - activity pub routes (question activities). Defaults to `30_000` (30 seconds). -## :hackney_pools +## HTTP client + +### :http + +* `proxy_url`: an upstream proxy to fetch posts and/or media with, (default: `nil`) +* `send_user_agent`: should we include a user agent with HTTP requests? (default: `true`) +* `user_agent`: what user agent should we use? (default: `:default`), must be string or `:default` +* `adapter`: array of hackney options + + +### :hackney_pools Advanced. Tweaks Hackney (http client) connections pools. From 02f7383891ff0a8dd17f00d6d00ec3495116e38a Mon Sep 17 00:00:00 2001 From: rinpatch Date: Mon, 25 Nov 2019 17:19:33 +0300 Subject: [PATCH 33/54] ActivityPub controller: do not render remote users --- .../web/activity_pub/activity_pub_controller.ex | 3 ++- .../activity_pub/activity_pub_controller_test.exs | 13 +++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index b2cd965fe..dec5da0d3 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -45,7 +45,7 @@ def relay_active?(conn, _) do end def user(conn, %{"nickname" => nickname}) do - with %User{} = user <- User.get_cached_by_nickname(nickname), + with %User{local: true} = user <- User.get_cached_by_nickname(nickname), {:ok, user} <- User.ensure_keys_present(user) do conn |> put_resp_content_type("application/activity+json") @@ -53,6 +53,7 @@ def user(conn, %{"nickname" => nickname}) do |> render("user.json", %{user: user}) else nil -> {:error, :not_found} + %{local: false} -> {:error, :not_found} end end diff --git a/test/web/activity_pub/activity_pub_controller_test.exs b/test/web/activity_pub/activity_pub_controller_test.exs index a5414c521..1aa73d75c 100644 --- a/test/web/activity_pub/activity_pub_controller_test.exs +++ b/test/web/activity_pub/activity_pub_controller_test.exs @@ -110,6 +110,19 @@ test "it returns a json representation of the user with accept application/ld+js assert json_response(conn, 200) == UserView.render("user.json", %{user: user}) end + + test "it returns 404 for remote users", %{ + conn: conn + } do + user = insert(:user, local: false, nickname: "remoteuser@example.com") + + conn = + conn + |> put_req_header("accept", "application/json") + |> get("/users/#{user.nickname}.json") + + assert json_response(conn, 404) + end end describe "/object/:uuid" do From 4b10804f21b80b74f9f9f85c9ecd6e1ec791255d Mon Sep 17 00:00:00 2001 From: rinpatch Date: Mon, 25 Nov 2019 17:55:17 +0300 Subject: [PATCH 34/54] OStatus controller: don't serve json at /notice/, redirect instead --- lib/pleroma/web/ostatus/ostatus_controller.ex | 46 ++++------- test/web/ostatus/ostatus_controller_test.exs | 79 ++++--------------- 2 files changed, 28 insertions(+), 97 deletions(-) diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex index 12a7c2365..01ec7941e 100644 --- a/lib/pleroma/web/ostatus/ostatus_controller.ex +++ b/lib/pleroma/web/ostatus/ostatus_controller.ex @@ -11,7 +11,6 @@ defmodule Pleroma.Web.OStatus.OStatusController do alias Pleroma.Plugs.RateLimiter alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPubController - alias Pleroma.Web.ActivityPub.ObjectView alias Pleroma.Web.ActivityPub.Visibility alias Pleroma.Web.Endpoint alias Pleroma.Web.Metadata.PlayerView @@ -38,11 +37,9 @@ def object(%{assigns: %{format: format}} = conn, %{"uuid" => uuid}) do with id <- o_status_url(conn, :object, uuid), {_, %Activity{} = activity} <- {:activity, Activity.get_create_by_object_ap_id_with_object(id)}, - {_, true} <- {:public?, Visibility.is_public?(activity)}, - %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do + {_, true} <- {:public?, Visibility.is_public?(activity)} do case format do - "html" -> redirect(conn, to: "/notice/#{activity.id}") - _ -> represent_activity(conn, nil, activity, user) + _ -> redirect(conn, to: "/notice/#{activity.id}") end else reason when reason in [{:public?, false}, {:activity, nil}] -> @@ -61,11 +58,9 @@ def activity(%{assigns: %{format: format}} = conn, %{"uuid" => _uuid}) def activity(%{assigns: %{format: format}} = conn, %{"uuid" => uuid}) do with id <- o_status_url(conn, :activity, uuid), {_, %Activity{} = activity} <- {:activity, Activity.normalize(id)}, - {_, true} <- {:public?, Visibility.is_public?(activity)}, - %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do + {_, true} <- {:public?, Visibility.is_public?(activity)} do case format do - "html" -> redirect(conn, to: "/notice/#{activity.id}") - _ -> represent_activity(conn, format, activity, user) + _ -> redirect(conn, to: "/notice/#{activity.id}") end else reason when reason in [{:public?, false}, {:activity, nil}] -> @@ -81,7 +76,15 @@ def notice(%{assigns: %{format: format}} = conn, %{"id" => id}) do {_, true} <- {:public?, Visibility.is_public?(activity)}, %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do cond do - format == "html" && activity.data["type"] == "Create" -> + format in ["json", "activity+json"] -> + if activity.local do + %{data: %{"id" => redirect_url}} = Object.normalize(activity) + redirect(conn, external: redirect_url) + else + {:error, :not_found} + end + + activity.data["type"] == "Create" -> %Object{} = object = Object.normalize(activity) RedirectController.redirector_with_meta( @@ -94,11 +97,8 @@ def notice(%{assigns: %{format: format}} = conn, %{"id" => id}) do } ) - format == "html" -> - RedirectController.redirector(conn, nil) - true -> - represent_activity(conn, format, activity, user) + RedirectController.redirector(conn, nil) end else reason when reason in [{:public?, false}, {:activity, nil}] -> @@ -135,24 +135,6 @@ def notice_player(conn, %{"id" => id}) do end end - defp represent_activity( - conn, - "activity+json", - %Activity{data: %{"type" => "Create"}} = activity, - _user - ) do - object = Object.normalize(activity) - - conn - |> put_resp_header("content-type", "application/activity+json") - |> put_view(ObjectView) - |> render("object.json", %{object: object}) - end - - defp represent_activity(_conn, _, _, _) do - {:error, :not_found} - end - def errors(conn, {:error, :not_found}) do render_error(conn, :not_found, "Not found") end diff --git a/test/web/ostatus/ostatus_controller_test.exs b/test/web/ostatus/ostatus_controller_test.exs index 37b7b62f5..50235dfef 100644 --- a/test/web/ostatus/ostatus_controller_test.exs +++ b/test/web/ostatus/ostatus_controller_test.exs @@ -35,23 +35,6 @@ test "redirects to /notice/id for html format", %{conn: conn} do assert redirected_to(conn) == "/notice/#{note_activity.id}" end - test "500s when user not found", %{conn: conn} do - note_activity = insert(:note_activity) - object = Object.normalize(note_activity) - user = User.get_cached_by_ap_id(note_activity.data["actor"]) - User.invalidate_cache(user) - Pleroma.Repo.delete(user) - [_, uuid] = hd(Regex.scan(~r/.+\/([\w-]+)$/, object.data["id"])) - url = "/objects/#{uuid}" - - conn = - conn - |> put_req_header("accept", "application/xml") - |> get(url) - - assert response(conn, 500) == ~S({"error":"Something went wrong"}) - end - test "404s on private objects", %{conn: conn} do note_activity = insert(:direct_note_activity) object = Object.normalize(note_activity) @@ -82,21 +65,6 @@ test "redirects to /notice/id for html format", %{conn: conn} do assert redirected_to(conn) == "/notice/#{note_activity.id}" end - test "505s when user not found", %{conn: conn} do - note_activity = insert(:note_activity) - [_, uuid] = hd(Regex.scan(~r/.+\/([\w-]+)$/, note_activity.data["id"])) - user = User.get_cached_by_ap_id(note_activity.data["actor"]) - User.invalidate_cache(user) - Pleroma.Repo.delete(user) - - conn = - conn - |> put_req_header("accept", "text/html") - |> get("/activities/#{uuid}") - - assert response(conn, 500) == ~S({"error":"Something went wrong"}) - end - test "404s on private activities", %{conn: conn} do note_activity = insert(:direct_note_activity) [_, uuid] = hd(Regex.scan(~r/.+\/([\w-]+)$/, note_activity.data["id"])) @@ -127,21 +95,28 @@ test "gets an activity in AS2 format", %{conn: conn} do end describe "GET notice/2" do - test "gets a notice in xml format", %{conn: conn} do + test "redirects to a proper object URL when json requested and the object is local", %{ + conn: conn + } do note_activity = insert(:note_activity) + expected_redirect_url = Object.normalize(note_activity).data["id"] - conn - |> get("/notice/#{note_activity.id}") - |> response(200) + redirect_url = + conn + |> put_req_header("accept", "application/activity+json") + |> get("/notice/#{note_activity.id}") + |> redirected_to() + + assert redirect_url == expected_redirect_url end - test "gets a notice in AS2 format", %{conn: conn} do - note_activity = insert(:note_activity) + test "returns a 404 on remote notice when json requested", %{conn: conn} do + note_activity = insert(:note_activity, local: false) conn |> put_req_header("accept", "application/activity+json") |> get("/notice/#{note_activity.id}") - |> json_response(200) + |> response(404) end test "500s when actor not found", %{conn: conn} do @@ -157,32 +132,6 @@ test "500s when actor not found", %{conn: conn} do assert response(conn, 500) == ~S({"error":"Something went wrong"}) end - test "only gets a notice in AS2 format for Create messages", %{conn: conn} do - note_activity = insert(:note_activity) - url = "/notice/#{note_activity.id}" - - conn = - conn - |> put_req_header("accept", "application/activity+json") - |> get(url) - - assert json_response(conn, 200) - - user = insert(:user) - - {:ok, like_activity, _} = CommonAPI.favorite(note_activity.id, user) - url = "/notice/#{like_activity.id}" - - assert like_activity.data["type"] == "Like" - - conn = - build_conn() - |> put_req_header("accept", "application/activity+json") - |> get(url) - - assert response(conn, 404) - end - test "render html for redirect for html format", %{conn: conn} do note_activity = insert(:note_activity) From f673e3deef8b8094a7a5fa012756c468901e5cac Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 26 Nov 2019 05:46:48 -0600 Subject: [PATCH 35/54] Default log level for syslog in prod should also be :warn --- config/prod.exs | 1 + 1 file changed, 1 insertion(+) diff --git a/config/prod.exs b/config/prod.exs index 301d2b9cb..25873f360 100644 --- a/config/prod.exs +++ b/config/prod.exs @@ -21,6 +21,7 @@ # Do not print debug messages in production config :logger, :console, level: :warn +config :logger, :ex_syslogger, level: :warn # ## SSL Support # From 80ededc04f4d4ba8318290976025b645a0ba3f95 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 26 Nov 2019 19:53:43 +0700 Subject: [PATCH 36/54] Add `direct_conversation_id` to web push payload --- lib/pleroma/activity.ex | 13 +++++++++++++ lib/pleroma/web/mastodon_api/views/status_view.ex | 10 ++-------- lib/pleroma/web/push/impl.ex | 6 +++++- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 7e283df32..cd7a5aae9 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -303,4 +303,17 @@ def restrict_deactivated_users(query) do end defdelegate search(user, query, options \\ []), to: Pleroma.Activity.Search + + def direct_conversation_id(activity, for_user) do + alias Pleroma.Conversation.Participation + + with %{data: %{"context" => context}} when is_binary(context) <- activity, + %Pleroma.Conversation{} = conversation <- Pleroma.Conversation.get_for_ap_id(context), + %Participation{id: participation_id} <- + Participation.for_user_and_conversation(for_user, conversation) do + participation_id + else + _ -> nil + end + end end diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index baff54151..a0257dfa6 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -9,8 +9,6 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do alias Pleroma.Activity alias Pleroma.ActivityExpiration - alias Pleroma.Conversation - alias Pleroma.Conversation.Participation alias Pleroma.HTML alias Pleroma.Object alias Pleroma.Repo @@ -245,12 +243,8 @@ def render("show.json", %{activity: %{data: %{"object" => _object}} = activity} direct_conversation_id = with {_, nil} <- {:direct_conversation_id, opts[:direct_conversation_id]}, {_, true} <- {:include_id, opts[:with_direct_conversation_id]}, - {_, %User{} = for_user} <- {:for_user, opts[:for]}, - %{data: %{"context" => context}} when is_binary(context) <- activity, - %Conversation{} = conversation <- Conversation.get_for_ap_id(context), - %Participation{id: participation_id} <- - Participation.for_user_and_conversation(for_user, conversation) do - participation_id + {_, %User{} = for_user} <- {:for_user, opts[:for]} do + Activity.direct_conversation_id(activity, for_user) else {:direct_conversation_id, participation_id} when is_integer(participation_id) -> participation_id diff --git a/lib/pleroma/web/push/impl.ex b/lib/pleroma/web/push/impl.ex index dd445e8bf..b01ae0f80 100644 --- a/lib/pleroma/web/push/impl.ex +++ b/lib/pleroma/web/push/impl.ex @@ -33,6 +33,8 @@ def perform( gcm_api_key = Application.get_env(:web_push_encryption, :gcm_api_key) avatar_url = User.avatar_url(actor) object = Object.normalize(activity) + user = User.get_cached_by_id(user_id) + direct_conversation_id = Activity.direct_conversation_id(activity, user) for subscription <- fetch_subsriptions(user_id), get_in(subscription.data, ["alerts", type]) do @@ -45,7 +47,9 @@ def perform( icon: avatar_url, preferred_locale: "en", pleroma: %{ - activity_id: activity_id + activity_id: activity_id, + direct_conversation_id: direct_conversation_id, + account: user.ap_id } } |> Jason.encode!() From 4af69f047db815a2349cadc757d5cff39174d83d Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 27 Nov 2019 02:32:55 +0700 Subject: [PATCH 37/54] Remove `account` field from web push payload --- lib/pleroma/web/push/impl.ex | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/pleroma/web/push/impl.ex b/lib/pleroma/web/push/impl.ex index b01ae0f80..3de7af708 100644 --- a/lib/pleroma/web/push/impl.ex +++ b/lib/pleroma/web/push/impl.ex @@ -48,8 +48,7 @@ def perform( preferred_locale: "en", pleroma: %{ activity_id: activity_id, - direct_conversation_id: direct_conversation_id, - account: user.ap_id + direct_conversation_id: direct_conversation_id } } |> Jason.encode!() From f01881e82a48b4177f943c73338d013010719040 Mon Sep 17 00:00:00 2001 From: Chris McCord Date: Thu, 11 Jul 2019 16:33:20 -0400 Subject: [PATCH 38/54] Add federated test infrastructure --- test/federation/federation_test.exs | 43 ++++++ test/support/cluster.ex | 218 ++++++++++++++++++++++++++++ 2 files changed, 261 insertions(+) create mode 100644 test/federation/federation_test.exs create mode 100644 test/support/cluster.ex diff --git a/test/federation/federation_test.exs b/test/federation/federation_test.exs new file mode 100644 index 000000000..7400d2abc --- /dev/null +++ b/test/federation/federation_test.exs @@ -0,0 +1,43 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2018 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Integration.FederationTest do + use Pleroma.DataCase + + import Pleroma.Cluster + + @federated1 :"federated1@127.0.0.1" + + describe "federated cluster primitives" do + test "within/2 captures local bindings and executes block on remote node" do + captured_binding = :captured + + result = + within @federated1 do + user = Pleroma.Factory.insert(:user) + {captured_binding, node(), user} + end + + assert {:captured, @federated1, user} = result + refute Pleroma.User.get_by_id(user.id) + assert user.id == within(@federated1, do: Pleroma.User.get_by_id(user.id)).id + end + + test "runs webserver on customized port" do + {nickname, url, url_404} = + within @federated1 do + import Pleroma.Web.Router.Helpers + user = Pleroma.Factory.insert(:user) + user_url = mastodon_api_url(Pleroma.Web.Endpoint, :user, user) + url_404 = mastodon_api_url(Pleroma.Web.Endpoint, :user, "not-exists") + + {user.nickname, user_url, url_404} + end + + assert {:ok, {{_, 200, _}, _headers, body}} = :httpc.request(~c"#{url}") + assert %{"acct" => ^nickname} = Jason.decode!(body) + assert {:ok, {{_, 404, _}, _headers, _body}} = :httpc.request(~c"#{url_404}") + end + end +end diff --git a/test/support/cluster.ex b/test/support/cluster.ex new file mode 100644 index 000000000..deb37f361 --- /dev/null +++ b/test/support/cluster.ex @@ -0,0 +1,218 @@ +defmodule Pleroma.Cluster do + @moduledoc """ + Facilities for managing a cluster of slave VM's for federated testing. + + ## Spawning the federated cluster + + `spawn_cluster/1` spawns a map of slave nodes that are started + within the running VM. During startup, the slave node is sent all configuration + from the parent node, as well as all code. After receiving configuration and + code, the slave then starts all applications currently running on the parent. + The configuration passed to `spawn_cluster/1` overrides any parent application + configuration for the provided OTP app and key. This is useful for customizing + the Ecto database, Phoenix webserver ports, etc. + + For example, to start a single federated VM named ":federated1", with the + Pleroma Endpoint running on port 4123, and with a database named + "pleroma_test1", you would run: + + endpoint_conf = Application.fetch_env!(:pleroma, Pleroma.Web.Endpoint) + repo_conf = Application.fetch_env!(:pleroma, Pleroma.Repo) + + Pleroma.Cluster.spawn_cluster(%{ + :"federated1@127.0.0.1" => [ + {:pleroma, Pleroma.Repo, Keyword.merge(repo_conf, database: "pleroma_test1")}, + {:pleroma, Pleroma.Web.Endpoint, + Keyword.merge(endpoint_conf, http: [port: 4011], url: [port: 4011], server: true)} + ] + }) + + *Note*: application configuration for a given key is not merged, + so any customization requires first fetching the existing values + and merging yourself by providing the merged configuration, + such as above with the endpoint config and repo config. + + ## Executing code within a remote node + + Use the `within/2` macro to execute code within the context of a remote + federated node. The code block captures all local variable bindings from + the parent's context and returns the result of the expression after executing + it on the remote node. For example: + + import Pleroma.Cluster + + parent_value = 123 + + result = + within :"federated1@127.0.0.1" do + {node(), parent_value} + end + + assert result == {:"federated1@127.0.0.1, 123} + + *Note*: while local bindings are captured and available within the block, + other parent contexts like required, aliased, or imported modules are not + in scope. Those will need to be reimported/aliases/required within the block + as `within/2` is a remote procedure call. + """ + + @extra_apps Pleroma.Mixfile.application()[:extra_applications] + + @doc """ + Spawns the default Pleroma federated cluster. + + Values before may be customized as needed for the test suite. + """ + def spawn_default_cluster do + endpoint_conf = Application.fetch_env!(:pleroma, Pleroma.Web.Endpoint) + repo_conf = Application.fetch_env!(:pleroma, Pleroma.Repo) + + spawn_cluster(%{ + :"federated1@127.0.0.1" => [ + {:pleroma, Pleroma.Repo, Keyword.merge(repo_conf, database: "pleroma_test_federated1")}, + {:pleroma, Pleroma.Web.Endpoint, + Keyword.merge(endpoint_conf, http: [port: 4011], url: [port: 4011], server: true)} + ], + :"federated2@127.0.0.1" => [ + {:pleroma, Pleroma.Repo, Keyword.merge(repo_conf, database: "pleroma_test_federated2")}, + {:pleroma, Pleroma.Web.Endpoint, + Keyword.merge(endpoint_conf, http: [port: 4012], url: [port: 4012], server: true)} + ] + }) + end + + @doc """ + Spawns a configured map of federated nodes. + + See `Pleroma.Cluster` module documentation for details. + """ + def spawn_cluster(node_configs) do + # Turn node into a distributed node with the given long name + :net_kernel.start([:"primary@127.0.0.1"]) + + # Allow spawned nodes to fetch all code from this node + {:ok, _} = :erl_boot_server.start([]) + allow_boot("127.0.0.1") + + silence_logger_warnings(fn -> + node_configs + |> Enum.map(&Task.async(fn -> start_slave(&1) end)) + |> Enum.map(&Task.await(&1, 60_000)) + end) + end + + @doc """ + Executes block of code again remote node. + + See `Pleroma.Cluster` module documentation for details. + """ + defmacro within(node, do: block) do + quote do + rpc(unquote(node), unquote(__MODULE__), :eval_quoted, [ + unquote(Macro.escape(block)), + binding() + ]) + end + end + + @doc false + def eval_quoted(block, binding) do + {result, _binding} = Code.eval_quoted(block, binding, __ENV__) + result + end + + defp start_slave({node_host, override_configs}) do + log(node_host, "booting federated VM") + {:ok, node} = :slave.start(~c"127.0.0.1", node_name(node_host), vm_args()) + add_code_paths(node) + load_apps_and_transfer_configuration(node, override_configs) + ensure_apps_started(node) + {:ok, node} + end + + def rpc(node, module, function, args) do + :rpc.block_call(node, module, function, args) + end + + defp vm_args do + ~c"-loader inet -hosts 127.0.0.1 -setcookie #{:erlang.get_cookie()}" + end + + defp allow_boot(host) do + {:ok, ipv4} = :inet.parse_ipv4_address(~c"#{host}") + :ok = :erl_boot_server.add_slave(ipv4) + end + + defp add_code_paths(node) do + rpc(node, :code, :add_paths, [:code.get_path()]) + end + + defp load_apps_and_transfer_configuration(node, override_configs) do + Enum.each(Application.loaded_applications(), fn {app_name, _, _} -> + app_name + |> Application.get_all_env() + |> Enum.each(fn {key, primary_config} -> + rpc(node, Application, :put_env, [app_name, key, primary_config, [persistent: true]]) + end) + end) + + Enum.each(override_configs, fn {app_name, key, val} -> + rpc(node, Application, :put_env, [app_name, key, val, [persistent: true]]) + end) + end + + defp log(node, msg), do: IO.puts("[#{node}] #{msg}") + + defp ensure_apps_started(node) do + loaded_names = Enum.map(Application.loaded_applications(), fn {name, _, _} -> name end) + app_names = @extra_apps ++ (loaded_names -- @extra_apps) + + rpc(node, Application, :ensure_all_started, [:mix]) + rpc(node, Mix, :env, [Mix.env()]) + rpc(node, __MODULE__, :prepare_database, []) + + log(node, "starting application") + + Enum.reduce(app_names, MapSet.new(), fn app, loaded -> + if Enum.member?(loaded, app) do + loaded + else + {:ok, started} = rpc(node, Application, :ensure_all_started, [app]) + MapSet.union(loaded, MapSet.new(started)) + end + end) + end + + @doc false + def prepare_database do + log(node(), "preparing database") + repo_config = Application.get_env(:pleroma, Pleroma.Repo) + repo_config[:adapter].storage_down(repo_config) + repo_config[:adapter].storage_up(repo_config) + + {:ok, _, _} = + Ecto.Migrator.with_repo(Pleroma.Repo, fn repo -> + Ecto.Migrator.run(repo, :up, log: false, all: true) + end) + + Ecto.Adapters.SQL.Sandbox.mode(Pleroma.Repo, :manual) + {:ok, _} = Application.ensure_all_started(:ex_machina) + end + + defp silence_logger_warnings(func) do + prev_level = Logger.level() + Logger.configure(level: :error) + res = func.() + Logger.configure(level: prev_level) + + res + end + + defp node_name(node_host) do + node_host + |> to_string() + |> String.split("@") + |> Enum.at(0) + |> String.to_atom() + end +end From 717d246491f65024957546fbbbaa7209d92d96d1 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 22 Jul 2019 16:17:18 +0200 Subject: [PATCH 39/54] Tests: Don't run federated tests by default --- test/federation/federation_test.exs | 8 ++++++-- test/test_helper.exs | 3 +-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/test/federation/federation_test.exs b/test/federation/federation_test.exs index 7400d2abc..8b8e8c870 100644 --- a/test/federation/federation_test.exs +++ b/test/federation/federation_test.exs @@ -4,11 +4,15 @@ defmodule Pleroma.Integration.FederationTest do use Pleroma.DataCase - + @moduletag :federated import Pleroma.Cluster - @federated1 :"federated1@127.0.0.1" + setup_all do + Pleroma.Cluster.spawn_default_cluster() + :ok + end + @federated1 :"federated1@127.0.0.1" describe "federated cluster primitives" do test "within/2 captures local bindings and executes block on remote node" do captured_binding = :captured diff --git a/test/test_helper.exs b/test/test_helper.exs index c8dbee010..b80c2683e 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -2,8 +2,7 @@ # Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -os_exclude = if :os.type() == {:unix, :darwin}, do: [skip_on_mac: true], else: [] -ExUnit.start(exclude: os_exclude) +ExUnit.start(exclude: [:federated]) Ecto.Adapters.SQL.Sandbox.mode(Pleroma.Repo, :manual) Mox.defmock(Pleroma.ReverseProxy.ClientMock, for: Pleroma.ReverseProxy.Client) {:ok, _} = Application.ensure_all_started(:ex_machina) From 74aa42998f23f49df1ea43564645ecbea3007c19 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 22 Jul 2019 16:18:37 +0200 Subject: [PATCH 40/54] CI: Add testing stage for federated tests. --- .gitlab-ci.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ab62c8827..78b560cac 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -55,6 +55,18 @@ unit-testing: - mix ecto.migrate - mix coveralls --preload-modules +federated-testing: + stage: test + services: + - name: lainsoykaf/postgres-with-rum + alias: postgres + command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] + script: + - mix deps.get + - mix ecto.create + - mix ecto.migrate + - mix test --trace --only federated + unit-testing-rum: stage: test services: From bbec9bb727baf6c76656b977fd8fbdd63aaaae07 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 22 Jul 2019 16:32:50 +0200 Subject: [PATCH 41/54] Tests: Run the daemon --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 78b560cac..66fa806f8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -65,6 +65,7 @@ federated-testing: - mix deps.get - mix ecto.create - mix ecto.migrate + - epmd -daemon - mix test --trace --only federated unit-testing-rum: From bd1984f054b075c04f2098afc5fe326fbc610c10 Mon Sep 17 00:00:00 2001 From: stwf Date: Tue, 26 Nov 2019 15:24:34 -0500 Subject: [PATCH 42/54] update tests and Oban --- mix.exs | 2 +- mix.lock | 8 ++++---- test/federation/federation_test.exs | 4 ++-- test/test_helper.exs | 4 +++- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/mix.exs b/mix.exs index 60c7fe3f6..eb2e54e4a 100644 --- a/mix.exs +++ b/mix.exs @@ -102,7 +102,7 @@ defp deps do {:phoenix_ecto, "~> 4.0"}, {:ecto_sql, "~> 3.2"}, {:postgrex, ">= 0.13.5"}, - {:oban, "~> 0.8.1"}, + {:oban, "~> 0.12.0"}, {:quantum, "~> 2.3"}, {:gettext, "~> 0.15"}, {:comeonin, "~> 4.1.1"}, diff --git a/mix.lock b/mix.lock index 3a664287c..0d41c4ad5 100644 --- a/mix.lock +++ b/mix.lock @@ -23,8 +23,8 @@ "decimal": {:hex, :decimal, "1.8.0", "ca462e0d885f09a1c5a342dbd7c1dcf27ea63548c65a65e67334f4b61803822e", [:mix], [], "hexpm"}, "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm"}, "earmark": {:hex, :earmark, "1.4.2", "3aa0bd23bc4c61cf2f1e5d752d1bb470560a6f8539974f767a38923bb20e1d7f", [:mix], [], "hexpm"}, - "ecto": {:hex, :ecto, "3.2.3", "51274df79862845b388733fddcf6f107d0c8c86e27abe7131fa98f8d30761bda", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, - "ecto_sql": {:hex, :ecto_sql, "3.2.0", "751cea597e8deb616084894dd75cbabfdbe7255ff01e8c058ca13f0353a3921b", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.2.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.2.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, + "ecto": {:hex, :ecto, "3.2.5", "76c864b77948a479e18e69cc1d0f0f4ee7cced1148ffe6a093ff91eba644f0b5", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, + "ecto_sql": {:hex, :ecto_sql, "3.2.2", "d10845bc147b9f61ef485cbf0973c0a337237199bd9bd30dd9542db00aadc26b", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.2.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.2.0 or ~> 0.3.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, "esshd": {:hex, :esshd, "0.1.0", "6f93a2062adb43637edad0ea7357db2702a4b80dd9683482fe00f5134e97f4c1", [:mix], [], "hexpm"}, "eternal": {:hex, :eternal, "1.2.1", "d5b6b2499ba876c57be2581b5b999ee9bdf861c647401066d3eeed111d096bc4", [:mix], [], "hexpm"}, "ex2ms": {:hex, :ex2ms, "1.5.0", "19e27f9212be9a96093fed8cdfbef0a2b56c21237196d26760f11dfcfae58e97", [:mix], [], "hexpm"}, @@ -67,7 +67,7 @@ "myhtmlex": {:git, "https://git.pleroma.social/pleroma/myhtmlex.git", "ad0097e2f61d4953bfef20fb6abddf23b87111e6", [ref: "ad0097e2f61d4953bfef20fb6abddf23b87111e6", submodules: true]}, "nimble_parsec": {:hex, :nimble_parsec, "0.5.1", "c90796ecee0289dbb5ad16d3ad06f957b0cd1199769641c961cfe0b97db190e0", [:mix], [], "hexpm"}, "nodex": {:git, "https://git.pleroma.social/pleroma/nodex", "cb6730f943cfc6aad674c92161be23a8411f15d1", [ref: "cb6730f943cfc6aad674c92161be23a8411f15d1"]}, - "oban": {:hex, :oban, "0.8.1", "4bbf62eb1829f856d69aeb5069ac7036afe07db8221a17de2a9169cc7a58a318", [:mix], [{:ecto_sql, "~> 3.1", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, + "oban": {:hex, :oban, "0.12.0", "5477d5ab4a5a201c0b6c89764040ebfc5d2c71c488a36f378016ce5990838f0f", [:mix], [{:ecto_sql, "~> 3.1", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm"}, "pbkdf2_elixir": {:hex, :pbkdf2_elixir, "0.12.4", "8dd29ed783f2e12195d7e0a4640effc0a7c37e6537da491f1db01839eee6d053", [:mix], [], "hexpm"}, "phoenix": {:hex, :phoenix, "1.4.10", "619e4a545505f562cd294df52294372d012823f4fd9d34a6657a8b242898c255", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.8.1 or ~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, @@ -97,7 +97,7 @@ "sweet_xml": {:hex, :sweet_xml, "0.6.6", "fc3e91ec5dd7c787b6195757fbcf0abc670cee1e4172687b45183032221b66b8", [:mix], [], "hexpm"}, "swoosh": {:hex, :swoosh, "0.23.5", "bfd9404bbf5069b1be2ffd317923ce57e58b332e25dbca2a35dedd7820dfee5a", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, - "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, + "telemetry": {:hex, :telemetry, "0.4.1", "ae2718484892448a24470e6aa341bc847c3277bfb8d4e9289f7474d752c09c7f", [:rebar3], [], "hexpm"}, "tesla": {:hex, :tesla, "1.3.0", "f35d72f029e608f9cdc6f6d6fcc7c66cf6d6512a70cfef9206b21b8bd0203a30", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, "~> 1.3", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "~> 4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 0.4", [hex: :mint, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.3", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm"}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, diff --git a/test/federation/federation_test.exs b/test/federation/federation_test.exs index 8b8e8c870..45800568a 100644 --- a/test/federation/federation_test.exs +++ b/test/federation/federation_test.exs @@ -33,8 +33,8 @@ test "runs webserver on customized port" do within @federated1 do import Pleroma.Web.Router.Helpers user = Pleroma.Factory.insert(:user) - user_url = mastodon_api_url(Pleroma.Web.Endpoint, :user, user) - url_404 = mastodon_api_url(Pleroma.Web.Endpoint, :user, "not-exists") + user_url = account_url(Pleroma.Web.Endpoint, :show, user) + url_404 = account_url(Pleroma.Web.Endpoint, :show, "not-exists") {user.nickname, user_url, url_404} end diff --git a/test/test_helper.exs b/test/test_helper.exs index b80c2683e..241ad1f94 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -2,7 +2,9 @@ # Copyright © 2017-2019 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -ExUnit.start(exclude: [:federated]) +os_exclude = if :os.type() == {:unix, :darwin}, do: [skip_on_mac: true], else: [] +ExUnit.start(exclude: [:federated | os_exclude]) + Ecto.Adapters.SQL.Sandbox.mode(Pleroma.Repo, :manual) Mox.defmock(Pleroma.ReverseProxy.ClientMock, for: Pleroma.ReverseProxy.Client) {:ok, _} = Application.ensure_all_started(:ex_machina) From 1fc28a4b441fdc0de8b2b43e2b045da38ee50b0a Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 26 Nov 2019 18:48:56 +0700 Subject: [PATCH 43/54] Add a view for the move notification --- docs/API/differences_in_mastoapi_responses.md | 6 +++ lib/pleroma/activity.ex | 3 +- .../mastodon_api/views/notification_view.ex | 38 ++++++++----------- lib/pleroma/web/push/impl.ex | 2 +- test/notification_test.exs | 29 ++++++++++++++ .../views/notification_view_test.exs | 24 ++++++++++++ 6 files changed, 77 insertions(+), 25 deletions(-) diff --git a/docs/API/differences_in_mastoapi_responses.md b/docs/API/differences_in_mastoapi_responses.md index e649bdf24..006d17c1b 100644 --- a/docs/API/differences_in_mastoapi_responses.md +++ b/docs/API/differences_in_mastoapi_responses.md @@ -92,6 +92,12 @@ Has these additional fields under the `pleroma` object: - `is_seen`: true if the notification was read by the user +### Move Notification + +The `type` value is `move`. Has an additional field: + +- `target`: new account + ## GET `/api/v1/notifications` Accepts additional parameters: diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index c1065611b..992298c2d 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -28,7 +28,8 @@ defmodule Pleroma.Activity do "Create" => "mention", "Follow" => "follow", "Announce" => "reblog", - "Like" => "favourite" + "Like" => "favourite", + "Move" => "move" } @mastodon_to_ap_notification_types for {k, v} <- @mastodon_notification_types, diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex index 5e3dbe728..ddd7f5318 100644 --- a/lib/pleroma/web/mastodon_api/views/notification_view.ex +++ b/lib/pleroma/web/mastodon_api/views/notification_view.ex @@ -37,32 +37,24 @@ def render("show.json", %{ } case mastodon_type do - "mention" -> - response - |> Map.merge(%{ - status: StatusView.render("show.json", %{activity: activity, for: user}) - }) - - "favourite" -> - response - |> Map.merge(%{ - status: StatusView.render("show.json", %{activity: parent_activity, for: user}) - }) - - "reblog" -> - response - |> Map.merge(%{ - status: StatusView.render("show.json", %{activity: parent_activity, for: user}) - }) - - "follow" -> - response - - _ -> - nil + "mention" -> put_status(response, activity, user) + "favourite" -> put_status(response, parent_activity, user) + "reblog" -> put_status(response, parent_activity, user) + "move" -> put_target(response, activity, user) + "follow" -> response + _ -> nil end else _ -> nil end end + + defp put_status(response, activity, user) do + Map.put(response, :status, StatusView.render("show.json", %{activity: activity, for: user})) + end + + defp put_target(response, activity, user) do + target = User.get_cached_by_ap_id(activity.data["target"]) + Map.put(response, :target, AccountView.render("show.json", %{user: target, for: user})) + end end diff --git a/lib/pleroma/web/push/impl.ex b/lib/pleroma/web/push/impl.ex index dd445e8bf..8c131409d 100644 --- a/lib/pleroma/web/push/impl.ex +++ b/lib/pleroma/web/push/impl.ex @@ -16,7 +16,7 @@ defmodule Pleroma.Web.Push.Impl do require Logger import Ecto.Query - @types ["Create", "Follow", "Announce", "Like"] + @types ["Create", "Follow", "Announce", "Like", "Move"] @doc "Performs sending notifications for user subscriptions" @spec perform(Notification.t()) :: list(any) | :error diff --git a/test/notification_test.exs b/test/notification_test.exs index f8d429223..dcbffeafe 100644 --- a/test/notification_test.exs +++ b/test/notification_test.exs @@ -630,6 +630,35 @@ test "notifications are deleted if a remote user is deleted" do assert Enum.empty?(Notification.for_user(local_user)) end + + test "move activity generates a notification" do + %{ap_id: old_ap_id} = old_user = insert(:user) + %{ap_id: new_ap_id} = new_user = insert(:user, also_known_as: [old_ap_id]) + follower = insert(:user) + other_follower = insert(:user, %{allow_following_move: false}) + + User.follow(follower, old_user) + User.follow(other_follower, old_user) + + Pleroma.Web.ActivityPub.ActivityPub.move(old_user, new_user) + ObanHelpers.perform_all() + + assert [ + %{ + activity: %{ + data: %{"type" => "Move", "actor" => ^old_ap_id, "target" => ^new_ap_id} + } + } + ] = Notification.for_user(follower) + + assert [ + %{ + activity: %{ + data: %{"type" => "Move", "actor" => ^old_ap_id, "target" => ^new_ap_id} + } + } + ] = Notification.for_user(other_follower) + end end describe "for_user" do diff --git a/test/web/mastodon_api/views/notification_view_test.exs b/test/web/mastodon_api/views/notification_view_test.exs index c9043a69a..80b6d414c 100644 --- a/test/web/mastodon_api/views/notification_view_test.exs +++ b/test/web/mastodon_api/views/notification_view_test.exs @@ -107,4 +107,28 @@ test "Follow notification" do assert [] == NotificationView.render("index.json", %{notifications: [notification], for: followed}) end + + test "Move notification" do + %{ap_id: old_ap_id} = old_user = insert(:user) + %{ap_id: _new_ap_id} = new_user = insert(:user, also_known_as: [old_ap_id]) + follower = insert(:user) + + User.follow(follower, old_user) + Pleroma.Web.ActivityPub.ActivityPub.move(old_user, new_user) + Pleroma.Tests.ObanHelpers.perform_all() + + [notification] = Notification.for_user(follower) + + expected = %{ + id: to_string(notification.id), + pleroma: %{is_seen: false}, + type: "move", + account: AccountView.render("show.json", %{user: old_user, for: follower}), + target: AccountView.render("show.json", %{user: new_user, for: follower}), + created_at: Utils.to_masto_date(notification.inserted_at) + } + + assert [expected] == + NotificationView.render("index.json", %{notifications: [notification], for: follower}) + end end From f595cfe6230715681e8ba93f73480bf695b3a243 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 27 Nov 2019 19:43:47 +0700 Subject: [PATCH 44/54] Remove User.user_info/2 --- lib/pleroma/user.ex | 25 ------ .../web/mastodon_api/views/account_view.ex | 11 ++- test/user_test.exs | 78 ++++--------------- .../mastodon_api/views/account_view_test.exs | 3 +- 4 files changed, 23 insertions(+), 94 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 3010fe87f..b18a4c6a5 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -177,20 +177,6 @@ def ap_followers(%User{} = user), do: "#{ap_id(user)}/followers" def ap_following(%User{following_address: fa}) when is_binary(fa), do: fa def ap_following(%User{} = user), do: "#{ap_id(user)}/following" - def user_info(%User{} = user, args \\ %{}) do - following_count = Map.get(args, :following_count, user.following_count) - follower_count = Map.get(args, :follower_count, user.follower_count) - - %{ - note_count: user.note_count, - locked: user.locked, - confirmation_pending: user.confirmation_pending, - default_scope: user.default_scope, - follower_count: follower_count, - following_count: following_count - } - end - def follow_state(%User{} = user, %User{} = target) do case Utils.fetch_latest_follow(user, target) do %{data: %{"state" => state}} -> state @@ -209,10 +195,6 @@ def set_follow_state_cache(user_ap_id, target_ap_id, state) do Cachex.put(:user_cache, "follow_state:#{user_ap_id}|#{target_ap_id}", state) end - def set_info_cache(user, args) do - Cachex.put(:user_cache, "user_info:#{user.id}", user_info(user, args)) - end - @spec restrict_deactivated(Ecto.Query.t()) :: Ecto.Query.t() def restrict_deactivated(query) do from(u in query, where: u.deactivated != ^true) @@ -614,7 +596,6 @@ def set_cache({:error, err}), do: {:error, err} def set_cache(%User{} = user) do Cachex.put(:user_cache, "ap_id:#{user.ap_id}", user) Cachex.put(:user_cache, "nickname:#{user.nickname}", user) - Cachex.put(:user_cache, "user_info:#{user.id}", user_info(user)) {:ok, user} end @@ -633,7 +614,6 @@ def update_and_set_cache(changeset) do def invalidate_cache(user) do Cachex.del(:user_cache, "ap_id:#{user.ap_id}") Cachex.del(:user_cache, "nickname:#{user.nickname}") - Cachex.del(:user_cache, "user_info:#{user.id}") end def get_cached_by_ap_id(ap_id) do @@ -701,11 +681,6 @@ def get_by_nickname_or_email(nickname_or_email) do get_by_nickname(nickname_or_email) || get_by_email(nickname_or_email) end - def get_cached_user_info(user) do - key = "user_info:#{user.id}" - Cachex.fetch!(:user_cache, key, fn -> user_info(user) end) - end - def fetch_by_nickname(nickname), do: ActivityPub.make_user_from_nickname(nickname) def get_or_fetch_by_nickname(nickname) do diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index e30fed610..7f50f3aa6 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -71,18 +71,17 @@ defp do_render("show.json", %{user: user} = opts) do image = User.avatar_url(user) |> MediaProxy.url() header = User.banner_url(user) |> MediaProxy.url() - user_info = User.get_cached_user_info(user) following_count = if !user.hide_follows_count or !user.hide_follows or opts[:for] == user do - user_info.following_count + user.following_count else 0 end followers_count = if !user.hide_followers_count or !user.hide_followers or opts[:for] == user do - user_info.follower_count + user.follower_count else 0 end @@ -144,7 +143,7 @@ defp do_render("show.json", %{user: user} = opts) do # Pleroma extension pleroma: %{ - confirmation_pending: user_info.confirmation_pending, + confirmation_pending: user.confirmation_pending, tags: user.tags, hide_followers_count: user.hide_followers_count, hide_follows_count: user.hide_follows_count, @@ -157,7 +156,7 @@ defp do_render("show.json", %{user: user} = opts) do } } |> maybe_put_role(user, opts[:for]) - |> maybe_put_settings(user, opts[:for], user_info) + |> maybe_put_settings(user, opts[:for], opts) |> maybe_put_notification_settings(user, opts[:for]) |> maybe_put_settings_store(user, opts[:for], opts) |> maybe_put_chat_token(user, opts[:for], opts) @@ -191,7 +190,7 @@ defp maybe_put_settings( data, %User{id: user_id} = user, %User{id: user_id}, - _user_info + _opts ) do data |> Kernel.put_in([:source, :privacy], user.default_scope) diff --git a/test/user_test.exs b/test/user_test.exs index e6302b525..82e338e75 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -961,9 +961,9 @@ test "hide a user from followers" do {:ok, user} = User.follow(user, user2) {:ok, _user} = User.deactivate(user) - info = User.get_cached_user_info(user2) + user2 = User.get_cached_by_id(user2.id) - assert info.follower_count == 0 + assert user2.follower_count == 0 assert [] = User.get_followers(user2) end @@ -977,10 +977,10 @@ test "hide a user from friends" do {:ok, _user} = User.deactivate(user) - info = User.get_cached_user_info(user2) + user2 = User.get_cached_by_id(user2.id) assert refresh_record(user2).following_count == 0 - assert info.following_count == 0 + assert user2.following_count == 0 assert User.following_count(user2) == 0 assert [] = User.get_friends(user2) end @@ -1182,13 +1182,12 @@ test "html_filter_policy returns TwitterText scrubber when rich-text is disabled describe "caching" do test "invalidate_cache works" do user = insert(:user) - _user_info = User.get_cached_user_info(user) + User.set_cache(user) User.invalidate_cache(user) {:ok, nil} = Cachex.get(:user_cache, "ap_id:#{user.ap_id}") {:ok, nil} = Cachex.get(:user_cache, "nickname:#{user.nickname}") - {:ok, nil} = Cachex.get(:user_cache, "user_info:#{user.id}") end test "User.delete() plugs any possible zombie objects" do @@ -1344,7 +1343,7 @@ test "follower count is updated when a follower is blocked" do {:ok, user} = User.block(user, follower) - assert User.user_info(user).follower_count == 2 + assert user.follower_count == 2 end describe "list_inactive_users_query/1" do @@ -1521,51 +1520,6 @@ test "external_users/1 external active users with limit", %{user1: user1, user2: end end - describe "set_info_cache/2" do - setup do - user = insert(:user) - {:ok, user: user} - end - - test "update from args", %{user: user} do - User.set_info_cache(user, %{following_count: 15, follower_count: 18}) - - %{follower_count: followers, following_count: following} = User.get_cached_user_info(user) - assert followers == 18 - assert following == 15 - end - - test "without args", %{user: user} do - User.set_info_cache(user, %{}) - - %{follower_count: followers, following_count: following} = User.get_cached_user_info(user) - assert followers == 0 - assert following == 0 - end - end - - describe "user_info/2" do - setup do - user = insert(:user) - {:ok, user: user} - end - - test "update from args", %{user: user} do - %{follower_count: followers, following_count: following} = - User.user_info(user, %{following_count: 15, follower_count: 18}) - - assert followers == 18 - assert following == 15 - end - - test "without args", %{user: user} do - %{follower_count: followers, following_count: following} = User.user_info(user) - - assert followers == 0 - assert following == 0 - end - end - describe "is_internal_user?/1" do test "non-internal user returns false" do user = insert(:user) @@ -1622,14 +1576,14 @@ test "updates the counters normally on following/getting a follow when disabled" ap_enabled: true ) - assert User.user_info(other_user).following_count == 0 - assert User.user_info(other_user).follower_count == 0 + assert other_user.following_count == 0 + assert other_user.follower_count == 0 {:ok, user} = Pleroma.User.follow(user, other_user) other_user = Pleroma.User.get_by_id(other_user.id) - assert User.user_info(user).following_count == 1 - assert User.user_info(other_user).follower_count == 1 + assert user.following_count == 1 + assert other_user.follower_count == 1 end test "syncronizes the counters with the remote instance for the followed when enabled" do @@ -1645,14 +1599,14 @@ test "syncronizes the counters with the remote instance for the followed when en ap_enabled: true ) - assert User.user_info(other_user).following_count == 0 - assert User.user_info(other_user).follower_count == 0 + assert other_user.following_count == 0 + assert other_user.follower_count == 0 Pleroma.Config.put([:instance, :external_user_synchronization], true) {:ok, _user} = User.follow(user, other_user) other_user = User.get_by_id(other_user.id) - assert User.user_info(other_user).follower_count == 437 + assert other_user.follower_count == 437 end test "syncronizes the counters with the remote instance for the follower when enabled" do @@ -1668,13 +1622,13 @@ test "syncronizes the counters with the remote instance for the follower when en ap_enabled: true ) - assert User.user_info(other_user).following_count == 0 - assert User.user_info(other_user).follower_count == 0 + assert other_user.following_count == 0 + assert other_user.follower_count == 0 Pleroma.Config.put([:instance, :external_user_synchronization], true) {:ok, other_user} = User.follow(other_user, user) - assert User.user_info(other_user).following_count == 152 + assert other_user.following_count == 152 end end diff --git a/test/web/mastodon_api/views/account_view_test.exs b/test/web/mastodon_api/views/account_view_test.exs index af88841ed..15bfcbb78 100644 --- a/test/web/mastodon_api/views/account_view_test.exs +++ b/test/web/mastodon_api/views/account_view_test.exs @@ -350,7 +350,8 @@ test "represent an embedded relationship" do } } - assert expected == AccountView.render("show.json", %{user: user, for: other_user}) + assert expected == + AccountView.render("show.json", %{user: refresh_record(user), for: other_user}) end test "returns the settings store if the requesting user is the represented user and it's requested specifically" do From f36724efb1fc0806e11b4c0c32a03d1aeee13da6 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 27 Nov 2019 14:13:36 +0100 Subject: [PATCH 45/54] User: Never return nil for user follower counts. --- lib/pleroma/user.ex | 4 ++-- test/user_test.exs | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 3010fe87f..16d0889c4 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -178,8 +178,8 @@ def ap_following(%User{following_address: fa}) when is_binary(fa), do: fa def ap_following(%User{} = user), do: "#{ap_id(user)}/following" def user_info(%User{} = user, args \\ %{}) do - following_count = Map.get(args, :following_count, user.following_count) - follower_count = Map.get(args, :follower_count, user.follower_count) + following_count = Map.get(args, :following_count, user.following_count) || 0 + follower_count = Map.get(args, :follower_count, user.follower_count) || 0 %{ note_count: user.note_count, diff --git a/test/user_test.exs b/test/user_test.exs index e6302b525..4917ea9ce 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -1347,6 +1347,14 @@ test "follower count is updated when a follower is blocked" do assert User.user_info(user).follower_count == 2 end + test "with nil follower count fields, 0 will be returned" do + user = insert(:user, follower_count: nil, following_count: nil) + user_info = User.user_info(user) + + assert user_info.follower_count == 0 + assert user_info.following_count == 0 + end + describe "list_inactive_users_query/1" do defp days_ago(days) do NaiveDateTime.add( From c4b468a95bd22dae16d89024016b2590e38d23ae Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 28 Nov 2019 16:44:48 +0700 Subject: [PATCH 46/54] Cleanup tests output --- test/web/activity_pub/transmogrifier_test.exs | 5 +++++ test/web/admin_api/admin_api_controller_test.exs | 1 + test/web/streamer/streamer_test.exs | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 0bdd514e9..b31c411dc 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -39,6 +39,7 @@ test "it ignores an incoming notice if we already have it" do assert activity == returned_activity end + @tag capture_log: true test "it fetches replied-to activities if we don't have them" do data = File.read!("test/fixtures/mastodon-post-activity.json") @@ -533,6 +534,7 @@ test "it works for incoming announces with an inlined activity" do assert object.data["content"] == "this is a private toot" end + @tag capture_log: true test "it rejects incoming announces with an inlined activity from another origin" do data = File.read!("test/fixtures/bogus-mastodon-announce.json") @@ -814,6 +816,7 @@ test "it fails for incoming deletes with spoofed origin" do assert Activity.get_by_id(activity.id) end + @tag capture_log: true test "it works for incoming user deletes" do %{ap_id: ap_id} = insert(:user, ap_id: "http://mastodon.example.org/users/admin") @@ -1749,6 +1752,7 @@ test "returns object with inReplyToAtomUri when denied incoming reply", %{data: assert modified_object["inReplyToAtomUri"] == "" end + @tag capture_log: true test "returns modified object when allowed incoming reply", %{data: data} do object_with_reply = Map.put( @@ -1868,6 +1872,7 @@ test "returns nil when cannot normalize object" do end) =~ "Unsupported URI scheme" end + @tag capture_log: true test "returns {:ok, %Object{}} for success case" do assert {:ok, %Object{}} = Transmogrifier.get_obj_helper("https://shitposter.club/notice/2827873") diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index bb2ca6a62..32577afee 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -1923,6 +1923,7 @@ test "with settings in db", %{conn: conn} do Pleroma.Config.put([:instance, :dynamic_configuration], true) end + @tag capture_log: true test "create new config setting in db", %{conn: conn} do conn = post(conn, "/api/pleroma/admin/config", %{ diff --git a/test/web/streamer/streamer_test.exs b/test/web/streamer/streamer_test.exs index 80a7541b2..8265f18dd 100644 --- a/test/web/streamer/streamer_test.exs +++ b/test/web/streamer/streamer_test.exs @@ -15,7 +15,7 @@ defmodule Pleroma.Web.StreamerTest do alias Pleroma.Web.Streamer.StreamerSocket alias Pleroma.Web.Streamer.Worker - @moduletag needs_streamer: true + @moduletag needs_streamer: true, capture_log: true clear_config_all([:instance, :skip_thread_containment]) describe "user streams" do From 9a2d38107ccaf8730a51ac052c73153d385a5cb6 Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 28 Nov 2019 15:14:11 +0100 Subject: [PATCH 47/54] CI: Make benchmarks manually triggered. --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ab62c8827..c4cde5cb9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,6 +31,7 @@ build: benchmark: stage: benchmark + when: manual variables: MIX_ENV: benchmark services: From f0bdbe3f61e3f3278f76d31dad447b982e2e2571 Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 28 Nov 2019 17:01:43 +0100 Subject: [PATCH 48/54] Migrations: Set users.following_count to NOT NULL Also set following_count for local users to the correct value and for remote users to 0. --- ...1128153944_fix_missing_following_count.exs | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 priv/repo/migrations/20191128153944_fix_missing_following_count.exs diff --git a/priv/repo/migrations/20191128153944_fix_missing_following_count.exs b/priv/repo/migrations/20191128153944_fix_missing_following_count.exs new file mode 100644 index 000000000..3236de7a4 --- /dev/null +++ b/priv/repo/migrations/20191128153944_fix_missing_following_count.exs @@ -0,0 +1,53 @@ +defmodule Pleroma.Repo.Migrations.FixMissingFollowingCount do + use Ecto.Migration + + def up do + """ + UPDATE + users + SET + following_count = sub.count + FROM + ( + SELECT + users.id AS sub_id + ,COUNT (following_relationships.id) + FROM + following_relationships + ,users + WHERE + users.id = following_relationships.follower_id + AND following_relationships.state = 'accept' + GROUP BY + users.id + ) AS sub + WHERE + users.id = sub.sub_id + AND users.local = TRUE + ; + """ + |> execute() + + """ + UPDATE + users + SET + following_count = 0 + WHERE + following_count IS NULL + """ + |> execute() + + execute("ALTER TABLE users + ALTER COLUMN following_count SET DEFAULT 0, + ALTER COLUMN following_count SET NOT NULL + ") + end + + def down do + execute("ALTER TABLE users + ALTER COLUMN following_count DROP DEFAULT, + ALTER COLUMN following_count DROP NOT NULL + ") + end +end From 798aa0c3350ba24b962c54bd1b9e5a14a70a76c0 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 28 Nov 2019 23:08:13 +0700 Subject: [PATCH 49/54] Use PosgreSQL v12 in the federated tests --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 66fa806f8..1a5df7f7e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -58,7 +58,7 @@ unit-testing: federated-testing: stage: test services: - - name: lainsoykaf/postgres-with-rum + - name: minibikini/postgres-with-rum:12 alias: postgres command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] script: From a98cda77580ead154dd2cb020c6d4ebc1e719d86 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Fri, 29 Nov 2019 15:49:35 +0700 Subject: [PATCH 50/54] Fix Pleroma.HTML.extract_first_external_url/2 --- lib/pleroma/html.ex | 1 + test/html_test.exs | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/lib/pleroma/html.ex b/lib/pleroma/html.ex index 997e965f0..4acd46253 100644 --- a/lib/pleroma/html.ex +++ b/lib/pleroma/html.ex @@ -91,6 +91,7 @@ def extract_first_external_url(object, content) do Cachex.fetch!(:scrubber_cache, key, fn _key -> result = content + |> HtmlEntities.decode() |> Floki.filter_out("a.mention,a.hashtag,a[rel~=\"tag\"]") |> Floki.attribute("a", "href") |> Enum.at(0) diff --git a/test/html_test.exs b/test/html_test.exs index f0869534c..c918dbe20 100644 --- a/test/html_test.exs +++ b/test/html_test.exs @@ -228,5 +228,16 @@ test "skips microformats hashtags" do assert url == "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=72255140" end + + test "does not crash when there is an HTML entity in a link" do + user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{"status" => "\"http://cofe.com/?boomer=ok&foo=bar\""}) + + object = Object.normalize(activity) + + assert {:ok, nil} = HTML.extract_first_external_url(object, object.data["content"]) + end end end From 0326683932800c20dcec2b7d3500bff149cf091b Mon Sep 17 00:00:00 2001 From: kPherox Date: Sat, 30 Nov 2019 04:26:45 +0900 Subject: [PATCH 51/54] hide follower/following for friendica --- lib/pleroma/web/activity_pub/activity_pub.ex | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index f25314ff6..f32d04175 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1233,13 +1233,13 @@ defp maybe_update_follow_information(data) do end end - defp collection_private(data) do - if is_map(data["first"]) and - data["first"]["type"] in ["CollectionPage", "OrderedCollectionPage"] do + defp collection_private(%{"first" => first}) do + if is_map(first) and + first["type"] in ["CollectionPage", "OrderedCollectionPage"] do {:ok, false} else with {:ok, %{"type" => type}} when type in ["CollectionPage", "OrderedCollectionPage"] <- - Fetcher.fetch_and_contain_remote_object_from_id(data["first"]) do + Fetcher.fetch_and_contain_remote_object_from_id(first) do {:ok, false} else {:error, {:ok, %{status: code}}} when code in [401, 403] -> @@ -1254,6 +1254,8 @@ defp collection_private(data) do end end + defp collection_private(_data), do: {:ok, true} + def user_data_from_user_object(data) do with {:ok, data} <- MRF.filter(data), {:ok, data} <- object_to_user_data(data) do From 1915b23e72c5cbed5c794f66eb0a727e719d3c57 Mon Sep 17 00:00:00 2001 From: kPherox Date: Sat, 30 Nov 2019 15:24:08 +0900 Subject: [PATCH 52/54] test missing first field --- .../users_mock/friendica_followers.json | 19 +++++++++++++++++++ .../users_mock/friendica_following.json | 19 +++++++++++++++++++ test/support/http_request_mock.ex | 16 ++++++++++++++++ test/web/activity_pub/activity_pub_test.exs | 15 +++++++++++++++ 4 files changed, 69 insertions(+) create mode 100644 test/fixtures/users_mock/friendica_followers.json create mode 100644 test/fixtures/users_mock/friendica_following.json diff --git a/test/fixtures/users_mock/friendica_followers.json b/test/fixtures/users_mock/friendica_followers.json new file mode 100644 index 000000000..7b86b5fe2 --- /dev/null +++ b/test/fixtures/users_mock/friendica_followers.json @@ -0,0 +1,19 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams", + "https://w3id.org/security/v1", + { + "vcard": "http://www.w3.org/2006/vcard/ns#", + "dfrn": "http://purl.org/macgirvin/dfrn/1.0/", + "diaspora": "https://diasporafoundation.org/ns/", + "litepub": "http://litepub.social/ns#", + "manuallyApprovesFollowers": "as:manuallyApprovesFollowers", + "sensitive": "as:sensitive", + "Hashtag": "as:Hashtag", + "directMessage": "litepub:directMessage" + } + ], + "id": "http://localhost:8080/followers/fuser3", + "type": "OrderedCollection", + "totalItems": 296 +} diff --git a/test/fixtures/users_mock/friendica_following.json b/test/fixtures/users_mock/friendica_following.json new file mode 100644 index 000000000..7c526befc --- /dev/null +++ b/test/fixtures/users_mock/friendica_following.json @@ -0,0 +1,19 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams", + "https://w3id.org/security/v1", + { + "vcard": "http://www.w3.org/2006/vcard/ns#", + "dfrn": "http://purl.org/macgirvin/dfrn/1.0/", + "diaspora": "https://diasporafoundation.org/ns/", + "litepub": "http://litepub.social/ns#", + "manuallyApprovesFollowers": "as:manuallyApprovesFollowers", + "sensitive": "as:sensitive", + "Hashtag": "as:Hashtag", + "directMessage": "litepub:directMessage" + } + ], + "id": "http://localhost:8080/following/fuser3", + "type": "OrderedCollection", + "totalItems": 32 +} diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex index 965335e96..e3a621f49 100644 --- a/test/support/http_request_mock.ex +++ b/test/support/http_request_mock.ex @@ -1035,6 +1035,22 @@ def get("http://localhost:4001/users/masto_closed/following?page=1", _, _, _) do }} end + def get("http://localhost:8080/followers/fuser3", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/users_mock/friendica_followers.json") + }} + end + + def get("http://localhost:8080/following/fuser3", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/users_mock/friendica_following.json") + }} + end + def get("http://localhost:4001/users/fuser2/followers", _, _, _) do {:ok, %Tesla.Env{ diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index d437ad456..30f5dde0c 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -1554,5 +1554,20 @@ test "detects hidden follows" do assert follow_info.hide_followers == false assert follow_info.hide_follows == true end + + test "detects hidden follows/followers for friendica" do + user = + insert(:user, + local: false, + follower_address: "http://localhost:8080/followers/fuser3", + following_address: "http://localhost:8080/following/fuser3" + ) + + {:ok, follow_info} = ActivityPub.fetch_follow_information_for_user(user) + assert follow_info.hide_followers == true + assert follow_info.follower_count == 296 + assert follow_info.following_count == 32 + assert follow_info.hide_follows == true + end end end From cf52106e059560afb50de5ae8044d0a51278fae1 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 2 Dec 2019 13:38:35 +0700 Subject: [PATCH 53/54] Update Floki dependency --- lib/pleroma/html.ex | 1 - mix.lock | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/pleroma/html.ex b/lib/pleroma/html.ex index 4acd46253..997e965f0 100644 --- a/lib/pleroma/html.ex +++ b/lib/pleroma/html.ex @@ -91,7 +91,6 @@ def extract_first_external_url(object, content) do Cachex.fetch!(:scrubber_cache, key, fn _key -> result = content - |> HtmlEntities.decode() |> Floki.filter_out("a.mention,a.hashtag,a[rel~=\"tag\"]") |> Floki.attribute("a", "href") |> Enum.at(0) diff --git a/mix.lock b/mix.lock index 0d41c4ad5..49bdad128 100644 --- a/mix.lock +++ b/mix.lock @@ -38,7 +38,7 @@ "fast_html": {:hex, :fast_html, "0.99.4", "d80812664f0429607e1d880fba0ef04da87a2e4fa596701bcaae17953535695c", [:make, :mix], [], "hexpm"}, "fast_sanitize": {:hex, :fast_sanitize, "0.1.4", "6c2e7203ca2f8275527a3021ba6e9d5d4ee213a47dc214a97c128737c9e56df1", [:mix], [{:fast_html, "~> 0.99", [hex: :fast_html, repo: "hexpm", optional: false]}, {:plug, "~> 1.8", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "flake_id": {:hex, :flake_id, "0.1.0", "7716b086d2e405d09b647121a166498a0d93d1a623bead243e1f74216079ccb3", [:mix], [{:base62, "~> 1.2", [hex: :base62, repo: "hexpm", optional: false]}, {:ecto, ">= 2.0.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"}, - "floki": {:hex, :floki, "0.23.0", "956ab6dba828c96e732454809fb0bd8d43ce0979b75f34de6322e73d4c917829", [:mix], [{:html_entities, "~> 0.4.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm"}, + "floki": {:hex, :floki, "0.23.1", "e100306ce7d8841d70a559748e5091542e2cfc67ffb3ade92b89a8435034dab1", [:mix], [{:html_entities, "~> 0.5.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm"}, "gen_smtp": {:hex, :gen_smtp, "0.15.0", "9f51960c17769b26833b50df0b96123605a8024738b62db747fece14eb2fbfcc", [:rebar3], [], "hexpm"}, "gen_stage": {:hex, :gen_stage, "0.14.3", "d0c66f1c87faa301c1a85a809a3ee9097a4264b2edf7644bf5c123237ef732bf", [:mix], [], "hexpm"}, "gen_state_machine": {:hex, :gen_state_machine, "2.0.5", "9ac15ec6e66acac994cc442dcc2c6f9796cf380ec4b08267223014be1c728a95", [:mix], [], "hexpm"}, From ed7f850ad2c9271268098959d8a86b24b187fa22 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Wed, 4 Dec 2019 16:22:01 +0300 Subject: [PATCH 54/54] fix test --- test/web/mastodon_api/views/notification_view_test.exs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/web/mastodon_api/views/notification_view_test.exs b/test/web/mastodon_api/views/notification_view_test.exs index 80b6d414c..26e1afc85 100644 --- a/test/web/mastodon_api/views/notification_view_test.exs +++ b/test/web/mastodon_api/views/notification_view_test.exs @@ -117,6 +117,9 @@ test "Move notification" do Pleroma.Web.ActivityPub.ActivityPub.move(old_user, new_user) Pleroma.Tests.ObanHelpers.perform_all() + old_user = refresh_record(old_user) + new_user = refresh_record(new_user) + [notification] = Notification.for_user(follower) expected = %{