From b655a8ea839d19443f44ff5b300a069d88ec7d58 Mon Sep 17 00:00:00 2001 From: href Date: Wed, 6 Feb 2019 10:33:05 +0100 Subject: [PATCH 01/19] Add recon --- mix.exs | 9 ++++++++- mix.lock | 8 ++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index 88da6332a..8cb389248 100644 --- a/mix.exs +++ b/mix.exs @@ -94,7 +94,14 @@ defp deps do {:auto_linker, git: "https://git.pleroma.social/pleroma/auto_linker.git", ref: "479dd343f4e563ff91215c8275f3b5c67e032850"}, - {:pleroma_job_queue, "~> 0.2.0"} + {:pleroma_job_queue, "~> 0.2.0"}, + {:telemetry, "~> 0.3"}, + {:prometheus_ex, "~> 3.0"}, + {:prometheus_plugs, "~> 1.1"}, + {:prometheus_phoenix, "~> 1.2"}, + {:prometheus_ecto, "~> 1.4"}, + {:prometheus_process_collector, "~> 1.4"}, + {:recon, github: "ferd/recon"} ] end diff --git a/mix.lock b/mix.lock index 9c454446a..0ece4b353 100644 --- a/mix.lock +++ b/mix.lock @@ -1,4 +1,5 @@ %{ + "accept": {:hex, :accept, "0.3.5", "b33b127abca7cc948bbe6caa4c263369abf1347cfa9d8e699c6d214660f10cd1", [:rebar3], [], "hexpm"}, "auto_linker": {:git, "https://git.pleroma.social/pleroma/auto_linker.git", "479dd343f4e563ff91215c8275f3b5c67e032850", [ref: "479dd343f4e563ff91215c8275f3b5c67e032850"]}, "base64url": {:hex, :base64url, "0.0.1", "36a90125f5948e3afd7be97662a1504b934dd5dac78451ca6e9abf85a10286be", [:rebar], [], "hexpm"}, "bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], [], "hexpm"}, @@ -57,7 +58,14 @@ "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm"}, "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm"}, "postgrex": {:hex, :postgrex, "0.14.1", "63247d4a5ad6b9de57a0bac5d807e1c32d41e39c04b8a4156a26c63bcd8a2e49", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, + "prometheus": {:hex, :prometheus, "4.2.2", "a830e77b79dc6d28183f4db050a7cac926a6c58f1872f9ef94a35cd989aceef8", [:mix, :rebar3], [], "hexpm"}, + "prometheus_ecto": {:hex, :prometheus_ecto, "1.4.1", "6c768ea9654de871e5b32fab2eac348467b3021604ebebbcbd8bcbe806a65ed5", [:mix], [{:ecto, "~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm"}, + "prometheus_ex": {:hex, :prometheus_ex, "3.0.5", "fa58cfd983487fc5ead331e9a3e0aa622c67232b3ec71710ced122c4c453a02f", [:mix], [{:prometheus, "~> 4.0", [hex: :prometheus, repo: "hexpm", optional: false]}], "hexpm"}, + "prometheus_phoenix": {:hex, :prometheus_phoenix, "1.2.1", "964a74dfbc055f781d3a75631e06ce3816a2913976d1df7830283aa3118a797a", [:mix], [{:phoenix, "~> 1.3", [hex: :phoenix, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.3 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm"}, + "prometheus_plugs": {:hex, :prometheus_plugs, "1.1.5", "25933d48f8af3a5941dd7b621c889749894d8a1082a6ff7c67cc99dec26377c5", [:mix], [{:accept, "~> 0.1", [hex: :accept, repo: "hexpm", optional: false]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}, {:prometheus_process_collector, "~> 1.1", [hex: :prometheus_process_collector, repo: "hexpm", optional: true]}], "hexpm"}, + "prometheus_process_collector": {:hex, :prometheus_process_collector, "1.4.0", "6dbd39e3165b9ef1c94a7a820e9ffe08479f949dcdd431ed4aaea7b250eebfde", [:rebar3], [{:prometheus, "~> 4.0", [hex: :prometheus, repo: "hexpm", optional: false]}], "hexpm"}, "ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm"}, + "recon": {:git, "https://github.com/ferd/recon.git", "75d70c7c08926d2f24f1ee6de14ee50fe8a52763", []}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm"}, "swoosh": {:hex, :swoosh, "0.20.0", "9a6c13822c9815993c03b6f8fccc370fcffb3c158d9754f67b1fdee6b3a5d928", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.12", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug, "~> 1.4", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, From bc3618a38d2e37254e27f723d3dd61679eca9be5 Mon Sep 17 00:00:00 2001 From: href Date: Wed, 30 Jan 2019 16:32:30 +0100 Subject: [PATCH 02/19] Set up telemetry and prometheus --- config/config.exs | 5 +++++ lib/pleroma/application.ex | 8 ++++++++ lib/pleroma/repo.ex | 4 ++++ lib/pleroma/web/endpoint.ex | 20 ++++++++++++++++++++ 4 files changed, 37 insertions(+) diff --git a/config/config.exs b/config/config.exs index dccf7b263..1e086f44c 100644 --- a/config/config.exs +++ b/config/config.exs @@ -8,6 +8,10 @@ # General application configuration config :pleroma, ecto_repos: [Pleroma.Repo] +config :pleroma, Pleroma.Repo, + types: Pleroma.PostgresTypes, + loggers: [Pleroma.Repo.Instrumenter, Ecto.LogEntry] + config :pleroma, Pleroma.Captcha, enabled: false, seconds_valid: 60, @@ -87,6 +91,7 @@ # Configures the endpoint config :pleroma, Pleroma.Web.Endpoint, + instrumenters: [Pleroma.Web.Endpoint.Instrumenter], url: [host: "localhost"], http: [ dispatch: [ diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 782d1d589..03dcbab1a 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -25,6 +25,7 @@ def start(_type, _args) do import Cachex.Spec Pleroma.Config.DeprecationWarnings.warn() + setup_instrumenters() # Define workers and child supervisors to be supervised children = @@ -140,6 +141,13 @@ def enabled_hackney_pools do end end + defp setup_instrumenters() do + Pleroma.Web.Endpoint.MetricsExporter.setup() + Pleroma.Web.Endpoint.PipelineInstrumenter.setup() + Pleroma.Web.Endpoint.Instrumenter.setup() + Pleroma.Repo.Instrumenter.setup() + end + if Mix.env() == :test do defp streamer_child, do: [] defp chat_child, do: [] diff --git a/lib/pleroma/repo.ex b/lib/pleroma/repo.ex index 4af1bde56..aa5d427ae 100644 --- a/lib/pleroma/repo.ex +++ b/lib/pleroma/repo.ex @@ -8,6 +8,10 @@ defmodule Pleroma.Repo do adapter: Ecto.Adapters.Postgres, migration_timestamps: [type: :naive_datetime_usec] + defmodule Instrumenter do + use Prometheus.EctoInstrumenter + end + @doc """ Dynamically loads the repository url from the DATABASE_URL environment variable. diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex index fa2d1cbe7..6d9528c86 100644 --- a/lib/pleroma/web/endpoint.ex +++ b/lib/pleroma/web/endpoint.ex @@ -70,6 +70,26 @@ defmodule Pleroma.Web.Endpoint do extra: "SameSite=Strict" ) + # Note: the plug and its configuration is compile-time this can't be upstreamed yet + if proxies = Pleroma.Config.get([__MODULE__, :reverse_proxies]) do + plug(RemoteIp, proxies: proxies) + end + + defmodule Instrumenter do + use Prometheus.PhoenixInstrumenter + end + + defmodule PipelineInstrumenter do + use Prometheus.PlugPipelineInstrumenter + end + + defmodule MetricsExporter do + use Prometheus.PlugExporter + end + + plug(PipelineInstrumenter) + plug(MetricsExporter) + plug(Pleroma.Web.Router) @doc """ From 0b5c818cb78b8c23fb2ba7ef372d0688ea9f36b7 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 25 Mar 2019 15:29:04 +0700 Subject: [PATCH 03/19] [#1] fix telemetry --- config/config.exs | 2 +- lib/pleroma/application.ex | 25 ++++++++++++++++++------- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/config/config.exs b/config/config.exs index 1e086f44c..4fd63f99d 100644 --- a/config/config.exs +++ b/config/config.exs @@ -10,7 +10,7 @@ config :pleroma, Pleroma.Repo, types: Pleroma.PostgresTypes, - loggers: [Pleroma.Repo.Instrumenter, Ecto.LogEntry] + telemetry_event: [Pleroma.Repo.Instrumenter] config :pleroma, Pleroma.Captcha, enabled: false, diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 03dcbab1a..c3f3126c6 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -127,6 +127,24 @@ def start(_type, _args) do Supervisor.start_link(children, opts) end + defp setup_instrumenters() do + require Prometheus.Registry + + :ok = + :telemetry.attach( + "prometheus-ecto", + [:pleroma, :repo, :query], + &Pleroma.Repo.Instrumenter.handle_event/4, + %{} + ) + + Prometheus.Registry.register_collector(:prometheus_process_collector) + Pleroma.Web.Endpoint.MetricsExporter.setup() + Pleroma.Web.Endpoint.PipelineInstrumenter.setup() + Pleroma.Web.Endpoint.Instrumenter.setup() + Pleroma.Repo.Instrumenter.setup() + end + def enabled_hackney_pools do [:media] ++ if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do @@ -141,13 +159,6 @@ def enabled_hackney_pools do end end - defp setup_instrumenters() do - Pleroma.Web.Endpoint.MetricsExporter.setup() - Pleroma.Web.Endpoint.PipelineInstrumenter.setup() - Pleroma.Web.Endpoint.Instrumenter.setup() - Pleroma.Repo.Instrumenter.setup() - end - if Mix.env() == :test do defp streamer_child, do: [] defp chat_child, do: [] From 5564cd421dfc706208df0b7447b0d692dffe052e Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 4 Apr 2019 12:19:31 -0500 Subject: [PATCH 04/19] Document Prometheus --- docs/api/prometheus.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 docs/api/prometheus.md diff --git a/docs/api/prometheus.md b/docs/api/prometheus.md new file mode 100644 index 000000000..19c564e3c --- /dev/null +++ b/docs/api/prometheus.md @@ -0,0 +1,22 @@ +# Prometheus Metrics + +Pleroma includes support for exporting metrics via the [prometheus_ex](https://github.com/deadtrickster/prometheus.ex) library. + +## `/api/pleroma/app_metrics` +### Exports Prometheus application metrics +* Method: `GET` +* Authentication: not required +* Params: none +* Response: JSON + +## Grafana +### Config example +The following is a config example to use with [Grafana](https://grafana.com) + +``` + - job_name: 'beam' + metrics_path: /api/pleroma/app_metrics + scheme: https + static_configs: + - targets: ['pleroma.soykaf.com'] +``` From 7e930559fece1a86891645333cc79a18f440ef1d Mon Sep 17 00:00:00 2001 From: href Date: Wed, 30 Jan 2019 16:44:38 +0100 Subject: [PATCH 05/19] Serve metrics at `/api/pleroma/app_metrics` --- config/config.exs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/config.exs b/config/config.exs index 4fd63f99d..ebaf1aec5 100644 --- a/config/config.exs +++ b/config/config.exs @@ -353,6 +353,7 @@ initial_timeout: 30, max_retries: 5 +<<<<<<< HEAD config :pleroma_job_queue, :queues, federator_incoming: 50, federator_outgoing: 50, @@ -385,6 +386,8 @@ config :pleroma, Pleroma.Mailer, adapter: Swoosh.Adapters.Sendmail +config :prometheus, Pleroma.Web.Endpoint.MetricsExporter, path: "/api/pleroma/app_metrics" + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" From 7222afe01b13586018b481172731309587191338 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 4 Apr 2019 12:29:10 -0500 Subject: [PATCH 06/19] Clean merge crumbs --- config/config.exs | 1 - 1 file changed, 1 deletion(-) diff --git a/config/config.exs b/config/config.exs index ebaf1aec5..b19b36b22 100644 --- a/config/config.exs +++ b/config/config.exs @@ -353,7 +353,6 @@ initial_timeout: 30, max_retries: 5 -<<<<<<< HEAD config :pleroma_job_queue, :queues, federator_incoming: 50, federator_outgoing: 50, From 69038887b2930072356aa00841b889c59518e264 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 4 Apr 2019 12:36:57 -0500 Subject: [PATCH 07/19] Code readability tweak --- lib/pleroma/application.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index c3f3126c6..1fc3fb728 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -127,7 +127,7 @@ def start(_type, _args) do Supervisor.start_link(children, opts) end - defp setup_instrumenters() do + defp setup_instrumenters do require Prometheus.Registry :ok = From f0f30019e1c9992cb420ba54457840cddaeb6a3a Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 5 Apr 2019 15:19:44 +0300 Subject: [PATCH 08/19] Refactor html caching functions to have a key instead of a module, use more correct terminology and fix summaries in mastoapi --- lib/pleroma/html.ex | 15 +++++++-------- lib/pleroma/web/mastodon_api/views/status_view.ex | 14 +++++++++++--- lib/pleroma/web/metadata/utils.ex | 2 +- .../web/twitter_api/views/activity_view.ex | 6 +++--- 4 files changed, 22 insertions(+), 15 deletions(-) diff --git a/lib/pleroma/html.ex b/lib/pleroma/html.ex index 1e48749a8..7f1dbe28c 100644 --- a/lib/pleroma/html.ex +++ b/lib/pleroma/html.ex @@ -28,21 +28,20 @@ def filter_tags(html, scrubber), do: Scrubber.scrub(html, scrubber) def filter_tags(html), do: filter_tags(html, nil) def strip_tags(html), do: Scrubber.scrub(html, Scrubber.StripTags) - # TODO: rename object to activity because that's what it is really working with - def get_cached_scrubbed_html_for_object(content, scrubbers, object, module) do - key = "#{module}#{generate_scrubber_signature(scrubbers)}|#{object.id}" + def get_cached_scrubbed_html_for_activity(content, scrubbers, activity, key \\ "") do + key = "#{key}#{generate_scrubber_signature(scrubbers)}|#{activity.id}" Cachex.fetch!(:scrubber_cache, key, fn _key -> - ensure_scrubbed_html(content, scrubbers, object.data["object"]["fake"] || false) + ensure_scrubbed_html(content, scrubbers, activity.data["object"]["fake"] || false) end) end - def get_cached_stripped_html_for_object(content, object, module) do - get_cached_scrubbed_html_for_object( + def get_cached_stripped_html_for_activity(content, activity, key) do + get_cached_scrubbed_html_for_activity( content, HtmlSanitizeEx.Scrubber.StripTags, - object, - module + activity, + key ) end diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 200bb453d..4c0b53bdd 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -147,10 +147,18 @@ def render("status.json", %{activity: %{data: %{"object" => object}} = activity} content = object |> render_content() - |> HTML.get_cached_scrubbed_html_for_object( + |> HTML.get_cached_scrubbed_html_for_activity( User.html_filter_policy(opts[:for]), activity, - __MODULE__ + "mastoapi:content" + ) + + summary = + (object["summary"] || "") + |> HTML.get_cached_scrubbed_html_for_activity( + User.html_filter_policy(opts[:for]), + activity, + "mastoapi:summary" ) card = render("card.json", Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)) @@ -182,7 +190,7 @@ def render("status.json", %{activity: %{data: %{"object" => object}} = activity} muted: CommonAPI.thread_muted?(user, activity) || User.mutes?(opts[:for], user), pinned: pinned?(activity, user), sensitive: sensitive, - spoiler_text: object["summary"] || "", + spoiler_text: summary, visibility: get_visibility(object), media_attachments: attachments, mentions: mentions, diff --git a/lib/pleroma/web/metadata/utils.ex b/lib/pleroma/web/metadata/utils.ex index 23bbde1a6..58385a3d1 100644 --- a/lib/pleroma/web/metadata/utils.ex +++ b/lib/pleroma/web/metadata/utils.ex @@ -12,7 +12,7 @@ def scrub_html_and_truncate(%{data: %{"content" => content}} = object) do # html content comes from DB already encoded, decode first and scrub after |> HtmlEntities.decode() |> String.replace(~r//, " ") - |> HTML.get_cached_stripped_html_for_object(object, __MODULE__) + |> HTML.get_cached_stripped_html_for_activity(object, "metadata") |> Formatter.demojify() |> Formatter.truncate() end diff --git a/lib/pleroma/web/twitter_api/views/activity_view.ex b/lib/pleroma/web/twitter_api/views/activity_view.ex index aa1d41fa2..433322eb8 100644 --- a/lib/pleroma/web/twitter_api/views/activity_view.ex +++ b/lib/pleroma/web/twitter_api/views/activity_view.ex @@ -254,10 +254,10 @@ def render( html = content - |> HTML.get_cached_scrubbed_html_for_object( + |> HTML.get_cached_scrubbed_html_for_activity( User.html_filter_policy(opts[:for]), activity, - __MODULE__ + "twitterapi:content" ) |> Formatter.emojify(object["emoji"]) @@ -265,7 +265,7 @@ def render( if content do content |> String.replace(~r//, "\n") - |> HTML.get_cached_stripped_html_for_object(activity, __MODULE__) + |> HTML.get_cached_stripped_html_for_activity(activity, "twitterapi:content") else "" end From 7895ee37fae82de26b3c06e69a96788d8c88d139 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sun, 16 Dec 2018 16:41:56 +0100 Subject: [PATCH 09/19] Add user following / unfollowing to the admin api. --- .../web/admin_api/admin_api_controller.ex | 20 ++++++++ lib/pleroma/web/router.ex | 4 ++ .../admin_api/admin_api_controller_test.exs | 46 +++++++++++++++++++ 3 files changed, 70 insertions(+) diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index b3a09e49e..84d0aabaf 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -25,6 +25,26 @@ def user_delete(conn, %{"nickname" => nickname}) do |> json(nickname) end + def user_follow(conn, %{"follower" => follower_nick, "followed" => followed_nick}) do + with %User{} = follower <- Repo.get_by(User, %{nickname: follower_nick}), + %User{} = followed <- Repo.get_by(User, %{nickname: followed_nick}) do + User.follow(follower, followed) + end + + conn + |> json("ok") + end + + def user_unfollow(conn, %{"follower" => follower_nick, "followed" => followed_nick}) do + with %User{} = follower <- Repo.get_by(User, %{nickname: follower_nick}), + %User{} = followed <- Repo.get_by(User, %{nickname: followed_nick}) do + User.unfollow(follower, followed) + end + + conn + |> json("ok") + end + def user_create( conn, %{"nickname" => nickname, "email" => email, "password" => password} diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 605a327fc..1c752e44c 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -140,8 +140,12 @@ defmodule Pleroma.Web.Router do scope "/api/pleroma/admin", Pleroma.Web.AdminAPI do pipe_through([:admin_api, :oauth_write]) + post("/user/follow", AdminAPIController, :user_follow) + post("/user/unfollow", AdminAPIController, :user_unfollow) + get("/users", AdminAPIController, :list_users) get("/users/:nickname", AdminAPIController, :user_show) + delete("/user", AdminAPIController, :user_delete) patch("/users/:nickname/toggle_activation", AdminAPIController, :user_toggle_activation) post("/user", AdminAPIController, :user_create) diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index acae64361..cedc907ec 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -74,6 +74,52 @@ test "when the user doesn't exist", %{conn: conn} do end end + describe "/api/pleroma/admin/user/follow" do + test "allows to force-follow another user" do + admin = insert(:user, info: %{is_admin: true}) + user = insert(:user) + follower = insert(:user) + + conn = + build_conn() + |> assign(:user, admin) + |> put_req_header("accept", "application/json") + |> post("/api/pleroma/admin/user/follow", %{ + "follower" => follower.nickname, + "followed" => user.nickname + }) + + user = Repo.get(User, user.id) + follower = Repo.get(User, follower.id) + + assert User.following?(follower, user) + end + end + + describe "/api/pleroma/admin/user/unfollow" do + test "allows to force-unfollow another user" do + admin = insert(:user, info: %{is_admin: true}) + user = insert(:user) + follower = insert(:user) + + User.follow(follower, user) + + conn = + build_conn() + |> assign(:user, admin) + |> put_req_header("accept", "application/json") + |> post("/api/pleroma/admin/user/unfollow", %{ + "follower" => follower.nickname, + "followed" => user.nickname + }) + + user = Repo.get(User, user.id) + follower = Repo.get(User, follower.id) + + refute User.following?(follower, user) + end + end + describe "PUT /api/pleroma/admin/users/tag" do setup do admin = insert(:user, info: %{is_admin: true}) From da64a5aece131d6bd8c0d17dcda61c626b44c4d0 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 5 Apr 2019 11:29:34 -0500 Subject: [PATCH 10/19] Document the admin API endpoints for controlling follow/unfollow --- docs/api/admin_api.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/api/admin_api.md b/docs/api/admin_api.md index 53b68ffd4..86cacebb1 100644 --- a/docs/api/admin_api.md +++ b/docs/api/admin_api.md @@ -58,6 +58,26 @@ Authentication is required and the user must be an admin. - `password` - Response: User’s nickname +## `/api/pleroma/admin/user/follow` +### Make a user follow another user + +- Methods: `POST` +- Params: + - `follower`: The nickname of the follower + - `followed`: The nickname of the followed +- Response: + - "ok" + +## `/api/pleroma/admin/user/unfollow` +### Make a user unfollow another user + +- Methods: `POST` +- Params: + - `follower`: The nickname of the follower + - `followed`: The nickname of the followed +- Response: + - "ok" + ## `/api/pleroma/admin/users/:nickname/toggle_activation` ### Toggle user activation From b5a2d384f71de9f7ff33d99c95c5db4674141d9a Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 5 Apr 2019 11:41:41 -0500 Subject: [PATCH 11/19] Redundant Repo.get_by usage was recently removed from the codebase --- lib/pleroma/web/admin_api/admin_api_controller.ex | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 84d0aabaf..78bf31893 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -26,8 +26,8 @@ def user_delete(conn, %{"nickname" => nickname}) do end def user_follow(conn, %{"follower" => follower_nick, "followed" => followed_nick}) do - with %User{} = follower <- Repo.get_by(User, %{nickname: follower_nick}), - %User{} = followed <- Repo.get_by(User, %{nickname: followed_nick}) do + with %User{} = follower <- User.get_by_nickname(follower_nick), + %User{} = followed <- User.get_by_nickname(followed_nick) do User.follow(follower, followed) end @@ -36,8 +36,8 @@ def user_follow(conn, %{"follower" => follower_nick, "followed" => followed_nick end def user_unfollow(conn, %{"follower" => follower_nick, "followed" => followed_nick}) do - with %User{} = follower <- Repo.get_by(User, %{nickname: follower_nick}), - %User{} = followed <- Repo.get_by(User, %{nickname: followed_nick}) do + with %User{} = follower <- User.get_by_nickname(follower_nick), + %User{} = followed <- User.get_by_nickname(followed_nick) do User.unfollow(follower, followed) end From c746087f570e366976b9b89c2aa6c2a5ff83c9ca Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 5 Apr 2019 11:59:56 -0500 Subject: [PATCH 12/19] Also remove Repo functions in the tests --- test/web/admin_api/admin_api_controller_test.exs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index cedc907ec..9c1cae6b7 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -89,8 +89,8 @@ test "allows to force-follow another user" do "followed" => user.nickname }) - user = Repo.get(User, user.id) - follower = Repo.get(User, follower.id) + user = User.get_by_nickname(user.id) + follower = User.get_by_nickname(follower.id) assert User.following?(follower, user) end @@ -113,8 +113,8 @@ test "allows to force-unfollow another user" do "followed" => user.nickname }) - user = Repo.get(User, user.id) - follower = Repo.get(User, follower.id) + user = User.get_by_nickname(user.id) + follower = User.get_by_nickname(follower.id) refute User.following?(follower, user) end From fac76bfa35f735005249111e74ea6be8670f5755 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 5 Apr 2019 12:11:19 -0500 Subject: [PATCH 13/19] We actually want the user id not nickname in the test... --- test/web/admin_api/admin_api_controller_test.exs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index 9c1cae6b7..dd2fbfb15 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -89,8 +89,8 @@ test "allows to force-follow another user" do "followed" => user.nickname }) - user = User.get_by_nickname(user.id) - follower = User.get_by_nickname(follower.id) + user = User.get_by_id(user.id) + follower = User.get_by_id(follower.id) assert User.following?(follower, user) end @@ -113,8 +113,8 @@ test "allows to force-unfollow another user" do "followed" => user.nickname }) - user = User.get_by_nickname(user.id) - follower = User.get_by_nickname(follower.id) + user = User.get_by_id(user.id) + follower = User.get_by_id(follower.id) refute User.following?(follower, user) end From fb1be1d79892a72b10af2c24479e81600603a6af Mon Sep 17 00:00:00 2001 From: optikfluffel Date: Fri, 5 Apr 2019 20:12:44 +0200 Subject: [PATCH 14/19] Use --cover option when running CI tests --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c07f1a5d3..0bd657d67 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -50,7 +50,7 @@ unit-testing: script: - mix ecto.create - mix ecto.migrate - - mix test --trace --preload-modules + - mix test --trace --preload-modules --cover lint: stage: test From e5df8cadeaa1ee3992e31e6ac00a0c391da7e4bd Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 5 Apr 2019 19:59:03 +0000 Subject: [PATCH 15/19] Revert "Merge branch 'test-coverage' into 'develop'" This reverts merge request !1027 --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0bd657d67..c07f1a5d3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -50,7 +50,7 @@ unit-testing: script: - mix ecto.create - mix ecto.migrate - - mix test --trace --preload-modules --cover + - mix test --trace --preload-modules lint: stage: test From e9c075d05c2f11b905d40ed86dd19818acf310ec Mon Sep 17 00:00:00 2001 From: Sergey Suprunenko Date: Fri, 5 Apr 2019 22:40:30 +0200 Subject: [PATCH 16/19] Mock :crypt.crypt/2 because otherwise the test fails on Mac OS --- test/plugs/legacy_authentication_plug_test.exs | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/test/plugs/legacy_authentication_plug_test.exs b/test/plugs/legacy_authentication_plug_test.exs index 302662797..8b0b06772 100644 --- a/test/plugs/legacy_authentication_plug_test.exs +++ b/test/plugs/legacy_authentication_plug_test.exs @@ -47,16 +47,18 @@ test "it authenticates the auth_user if present and password is correct and rese |> assign(:auth_user, user) conn = - with_mock User, - reset_password: fn user, %{password: password, password_confirmation: password} -> - send(self(), :reset_password) - {:ok, user} - end do - conn - |> LegacyAuthenticationPlug.call(%{}) + with_mocks([ + {:crypt, [], [crypt: fn _password, password_hash -> password_hash end]}, + {User, [], + [ + reset_password: fn user, %{password: password, password_confirmation: password} -> + {:ok, user} + end + ]} + ]) do + LegacyAuthenticationPlug.call(conn, %{}) end - assert_received :reset_password assert conn.assigns.user == user end From 325a2680173f714a5875ed726f9171e7984f7f7a Mon Sep 17 00:00:00 2001 From: Sergey Suprunenko Date: Fri, 5 Apr 2019 23:36:42 +0000 Subject: [PATCH 17/19] Redirect to the referer url after mastofe authorization --- .../mastodon_api/mastodon_api_controller.ex | 19 ++++-- test/support/factory.ex | 10 +++ .../mastodon_api_controller_test.exs | 67 +++++++++++++++++++ 3 files changed, 90 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 89fd7629a..bcc79b08a 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -1091,9 +1091,7 @@ def list_timeline(%{assigns: %{user: user}} = conn, %{"list_id" => id} = params) end def index(%{assigns: %{user: user}} = conn, _params) do - token = - conn - |> get_session(:oauth_token) + token = get_session(conn, :oauth_token) if user && token do mastodon_emoji = mastodonized_emoji() @@ -1194,6 +1192,7 @@ def index(%{assigns: %{user: user}} = conn, _params) do |> render("index.html", %{initial_state: initial_state, flavour: flavour}) else conn + |> put_session(:return_to, conn.request_path) |> redirect(to: "/web/login") end end @@ -1278,12 +1277,20 @@ def login(conn, _) do scope: Enum.join(app.scopes, " ") ) - conn - |> redirect(to: path) + redirect(conn, to: path) end end - defp local_mastodon_root_path(conn), do: mastodon_api_path(conn, :index, ["getting-started"]) + defp local_mastodon_root_path(conn) do + case get_session(conn, :return_to) do + nil -> + mastodon_api_path(conn, :index, ["getting-started"]) + + return_to -> + delete_session(conn, :return_to) + return_to + end + end defp get_or_make_app do find_attrs = %{client_name: @local_mastodon_name, redirect_uris: "."} diff --git a/test/support/factory.ex b/test/support/factory.ex index e1a08315a..b37bc2c07 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -240,6 +240,16 @@ def oauth_token_factory do } end + def oauth_authorization_factory do + %Pleroma.Web.OAuth.Authorization{ + token: :crypto.strong_rand_bytes(32) |> Base.url_encode64(padding: false), + scopes: ["read", "write", "follow", "push"], + valid_until: NaiveDateTime.add(NaiveDateTime.utc_now(), 60 * 10), + user: build(:user), + app: build(:oauth_app) + } + end + def push_subscription_factory do %Pleroma.Web.Push.Subscription{ user: build(:user), diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 6060cc97f..438e9507d 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -2340,4 +2340,71 @@ test "accounts fetches correct account for nicknames beginning with numbers", %{ refute acc_one == acc_two assert acc_two == acc_three end + + describe "index/2 redirections" do + setup %{conn: conn} do + session_opts = [ + store: :cookie, + key: "_test", + signing_salt: "cooldude" + ] + + conn = + conn + |> Plug.Session.call(Plug.Session.init(session_opts)) + |> fetch_session() + + test_path = "/web/statuses/test" + %{conn: conn, path: test_path} + end + + test "redirects not logged-in users to the login page", %{conn: conn, path: path} do + conn = get(conn, path) + + assert conn.status == 302 + assert redirected_to(conn) == "/web/login" + end + + test "does not redirect logged in users to the login page", %{conn: conn, path: path} do + token = insert(:oauth_token) + + conn = + conn + |> assign(:user, token.user) + |> put_session(:oauth_token, token.token) + |> get(path) + + assert conn.status == 200 + end + + test "saves referer path to session", %{conn: conn, path: path} do + conn = get(conn, path) + return_to = Plug.Conn.get_session(conn, :return_to) + + assert return_to == path + end + + test "redirects to the saved path after log in", %{conn: conn, path: path} do + app = insert(:oauth_app, client_name: "Mastodon-Local", redirect_uris: ".") + auth = insert(:oauth_authorization, app: app) + + conn = + conn + |> put_session(:return_to, path) + |> get("/web/login", %{code: auth.token}) + + assert conn.status == 302 + assert redirected_to(conn) == path + end + + test "redirects to the getting-started page when referer is not present", %{conn: conn} do + app = insert(:oauth_app, client_name: "Mastodon-Local", redirect_uris: ".") + auth = insert(:oauth_authorization, app: app) + + conn = get(conn, "/web/login", %{code: auth.token}) + + assert conn.status == 302 + assert redirected_to(conn) == "/web/getting-started" + end + end end From b395aebf2489f44bdb1a9c4905a51f0f26bf5fab Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 6 Apr 2019 09:30:36 -0500 Subject: [PATCH 18/19] Pin recon dependency to 2.4.0 --- mix.exs | 2 +- mix.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index 6e7cff413..ec0865c4f 100644 --- a/mix.exs +++ b/mix.exs @@ -101,7 +101,7 @@ defp deps do {:prometheus_phoenix, "~> 1.2"}, {:prometheus_ecto, "~> 1.4"}, {:prometheus_process_collector, "~> 1.4"}, - {:recon, github: "ferd/recon"}, + {:recon, github: "ferd/recon", tag: "2.4.0"}, {:quack, "~> 0.1.1"} ] end diff --git a/mix.lock b/mix.lock index 662fd0c6e..7c7e322de 100644 --- a/mix.lock +++ b/mix.lock @@ -66,7 +66,7 @@ "prometheus_process_collector": {:hex, :prometheus_process_collector, "1.4.0", "6dbd39e3165b9ef1c94a7a820e9ffe08479f949dcdd431ed4aaea7b250eebfde", [:rebar3], [{:prometheus, "~> 4.0", [hex: :prometheus, repo: "hexpm", optional: false]}], "hexpm"}, "quack": {:hex, :quack, "0.1.1", "cca7b4da1a233757fdb44b3334fce80c94785b3ad5a602053b7a002b5a8967bf", [:mix], [{:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: false]}, {:tesla, "~> 1.2.0", [hex: :tesla, repo: "hexpm", optional: false]}], "hexpm"}, "ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm"}, - "recon": {:git, "https://github.com/ferd/recon.git", "75d70c7c08926d2f24f1ee6de14ee50fe8a52763", []}, + "recon": {:git, "https://github.com/ferd/recon.git", "75d70c7c08926d2f24f1ee6de14ee50fe8a52763", [tag: "2.4.0"]}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm"}, "swoosh": {:hex, :swoosh, "0.20.0", "9a6c13822c9815993c03b6f8fccc370fcffb3c158d9754f67b1fdee6b3a5d928", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.12", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug, "~> 1.4", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, From 7aa53d52bd982b5ab233a65048f5fb1823127d4a Mon Sep 17 00:00:00 2001 From: eugenijm Date: Sat, 6 Apr 2019 00:22:42 +0300 Subject: [PATCH 19/19] Return 403 on oauth token exchange for a deactivated user --- lib/pleroma/web/oauth/oauth_controller.ex | 6 ++++++ test/web/oauth/oauth_controller_test.exs | 26 +++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/lib/pleroma/web/oauth/oauth_controller.ex b/lib/pleroma/web/oauth/oauth_controller.ex index 26d53df1a..aac8f97fc 100644 --- a/lib/pleroma/web/oauth/oauth_controller.ex +++ b/lib/pleroma/web/oauth/oauth_controller.ex @@ -152,6 +152,7 @@ def token_exchange( with {_, {:ok, %User{} = user}} <- {:get_user, Authenticator.get_user(conn)}, %App{} = app <- get_app_from_request(conn, params), {:auth_active, true} <- {:auth_active, User.auth_active?(user)}, + {:user_active, true} <- {:user_active, !user.info.deactivated}, scopes <- oauth_scopes(params, app.scopes), [] <- scopes -- app.scopes, true <- Enum.any?(scopes), @@ -175,6 +176,11 @@ def token_exchange( |> put_status(:forbidden) |> json(%{error: "Your login is missing a confirmed e-mail address"}) + {:user_active, false} -> + conn + |> put_status(:forbidden) + |> json(%{error: "Your account is currently disabled"}) + _error -> put_status(conn, 400) |> json(%{error: "Invalid credentials"}) diff --git a/test/web/oauth/oauth_controller_test.exs b/test/web/oauth/oauth_controller_test.exs index a9a0b9ed4..a68528420 100644 --- a/test/web/oauth/oauth_controller_test.exs +++ b/test/web/oauth/oauth_controller_test.exs @@ -327,6 +327,32 @@ test "rejects token exchange for valid credentials belonging to unconfirmed user refute Map.has_key?(resp, "access_token") end + test "rejects token exchange for valid credentials belonging to deactivated user" do + password = "testpassword" + + user = + insert(:user, + password_hash: Comeonin.Pbkdf2.hashpwsalt(password), + info: %{deactivated: true} + ) + + app = insert(:oauth_app) + + conn = + build_conn() + |> post("/oauth/token", %{ + "grant_type" => "password", + "username" => user.nickname, + "password" => password, + "client_id" => app.client_id, + "client_secret" => app.client_secret + }) + + assert resp = json_response(conn, 403) + assert %{"error" => _} = resp + refute Map.has_key?(resp, "access_token") + end + test "rejects an invalid authorization code" do app = insert(:oauth_app)