Merge branch 'develop' into use-jobs-in-webpush

This commit is contained in:
Egor Kislitsyn 2019-04-08 16:29:20 +07:00
commit a1a854646e
21 changed files with 331 additions and 29 deletions

View File

@ -8,6 +8,10 @@
# General application configuration
config :pleroma, ecto_repos: [Pleroma.Repo]
config :pleroma, Pleroma.Repo,
types: Pleroma.PostgresTypes,
telemetry_event: [Pleroma.Repo.Instrumenter]
config :pleroma, Pleroma.Captcha,
enabled: false,
seconds_valid: 60,
@ -87,6 +91,7 @@
# Configures the endpoint
config :pleroma, Pleroma.Web.Endpoint,
instrumenters: [Pleroma.Web.Endpoint.Instrumenter],
url: [host: "localhost"],
http: [
dispatch: [
@ -386,6 +391,8 @@
config :pleroma, Pleroma.Mailer, adapter: Swoosh.Adapters.Sendmail
config :prometheus, Pleroma.Web.Endpoint.MetricsExporter, path: "/api/pleroma/app_metrics"
# Import environment specific config. This must remain at the bottom
# of this file so it overrides the configuration defined above.
import_config "#{Mix.env()}.exs"

View File

@ -58,6 +58,26 @@ Authentication is required and the user must be an admin.
- `password`
- Response: Users nickname
## `/api/pleroma/admin/user/follow`
### Make a user follow another user
- Methods: `POST`
- Params:
- `follower`: The nickname of the follower
- `followed`: The nickname of the followed
- Response:
- "ok"
## `/api/pleroma/admin/user/unfollow`
### Make a user unfollow another user
- Methods: `POST`
- Params:
- `follower`: The nickname of the follower
- `followed`: The nickname of the followed
- Response:
- "ok"
## `/api/pleroma/admin/users/:nickname/toggle_activation`
### Toggle user activation

22
docs/api/prometheus.md Normal file
View File

@ -0,0 +1,22 @@
# Prometheus Metrics
Pleroma includes support for exporting metrics via the [prometheus_ex](https://github.com/deadtrickster/prometheus.ex) library.
## `/api/pleroma/app_metrics`
### Exports Prometheus application metrics
* Method: `GET`
* Authentication: not required
* Params: none
* Response: JSON
## Grafana
### Config example
The following is a config example to use with [Grafana](https://grafana.com)
```
- job_name: 'beam'
metrics_path: /api/pleroma/app_metrics
scheme: https
static_configs:
- targets: ['pleroma.soykaf.com']
```

View File

@ -25,6 +25,7 @@ def start(_type, _args) do
import Cachex.Spec
Pleroma.Config.DeprecationWarnings.warn()
setup_instrumenters()
# Define workers and child supervisors to be supervised
children =
@ -126,6 +127,24 @@ def start(_type, _args) do
Supervisor.start_link(children, opts)
end
defp setup_instrumenters do
require Prometheus.Registry
:ok =
:telemetry.attach(
"prometheus-ecto",
[:pleroma, :repo, :query],
&Pleroma.Repo.Instrumenter.handle_event/4,
%{}
)
Prometheus.Registry.register_collector(:prometheus_process_collector)
Pleroma.Web.Endpoint.MetricsExporter.setup()
Pleroma.Web.Endpoint.PipelineInstrumenter.setup()
Pleroma.Web.Endpoint.Instrumenter.setup()
Pleroma.Repo.Instrumenter.setup()
end
def enabled_hackney_pools do
[:media] ++
if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do

View File

@ -28,21 +28,20 @@ def filter_tags(html, scrubber), do: Scrubber.scrub(html, scrubber)
def filter_tags(html), do: filter_tags(html, nil)
def strip_tags(html), do: Scrubber.scrub(html, Scrubber.StripTags)
# TODO: rename object to activity because that's what it is really working with
def get_cached_scrubbed_html_for_object(content, scrubbers, object, module) do
key = "#{module}#{generate_scrubber_signature(scrubbers)}|#{object.id}"
def get_cached_scrubbed_html_for_activity(content, scrubbers, activity, key \\ "") do
key = "#{key}#{generate_scrubber_signature(scrubbers)}|#{activity.id}"
Cachex.fetch!(:scrubber_cache, key, fn _key ->
ensure_scrubbed_html(content, scrubbers, object.data["object"]["fake"] || false)
ensure_scrubbed_html(content, scrubbers, activity.data["object"]["fake"] || false)
end)
end
def get_cached_stripped_html_for_object(content, object, module) do
get_cached_scrubbed_html_for_object(
def get_cached_stripped_html_for_activity(content, activity, key) do
get_cached_scrubbed_html_for_activity(
content,
HtmlSanitizeEx.Scrubber.StripTags,
object,
module
activity,
key
)
end

View File

@ -8,6 +8,10 @@ defmodule Pleroma.Repo do
adapter: Ecto.Adapters.Postgres,
migration_timestamps: [type: :naive_datetime_usec]
defmodule Instrumenter do
use Prometheus.EctoInstrumenter
end
@doc """
Dynamically loads the repository url from the
DATABASE_URL environment variable.

View File

@ -25,6 +25,26 @@ def user_delete(conn, %{"nickname" => nickname}) do
|> json(nickname)
end
def user_follow(conn, %{"follower" => follower_nick, "followed" => followed_nick}) do
with %User{} = follower <- User.get_by_nickname(follower_nick),
%User{} = followed <- User.get_by_nickname(followed_nick) do
User.follow(follower, followed)
end
conn
|> json("ok")
end
def user_unfollow(conn, %{"follower" => follower_nick, "followed" => followed_nick}) do
with %User{} = follower <- User.get_by_nickname(follower_nick),
%User{} = followed <- User.get_by_nickname(followed_nick) do
User.unfollow(follower, followed)
end
conn
|> json("ok")
end
def user_create(
conn,
%{"nickname" => nickname, "email" => email, "password" => password}

View File

@ -70,6 +70,26 @@ defmodule Pleroma.Web.Endpoint do
extra: "SameSite=Strict"
)
# Note: the plug and its configuration is compile-time this can't be upstreamed yet
if proxies = Pleroma.Config.get([__MODULE__, :reverse_proxies]) do
plug(RemoteIp, proxies: proxies)
end
defmodule Instrumenter do
use Prometheus.PhoenixInstrumenter
end
defmodule PipelineInstrumenter do
use Prometheus.PlugPipelineInstrumenter
end
defmodule MetricsExporter do
use Prometheus.PlugExporter
end
plug(PipelineInstrumenter)
plug(MetricsExporter)
plug(Pleroma.Web.Router)
@doc """

View File

@ -1091,9 +1091,7 @@ def list_timeline(%{assigns: %{user: user}} = conn, %{"list_id" => id} = params)
end
def index(%{assigns: %{user: user}} = conn, _params) do
token =
conn
|> get_session(:oauth_token)
token = get_session(conn, :oauth_token)
if user && token do
mastodon_emoji = mastodonized_emoji()
@ -1194,6 +1192,7 @@ def index(%{assigns: %{user: user}} = conn, _params) do
|> render("index.html", %{initial_state: initial_state, flavour: flavour})
else
conn
|> put_session(:return_to, conn.request_path)
|> redirect(to: "/web/login")
end
end
@ -1278,12 +1277,20 @@ def login(conn, _) do
scope: Enum.join(app.scopes, " ")
)
conn
|> redirect(to: path)
redirect(conn, to: path)
end
end
defp local_mastodon_root_path(conn), do: mastodon_api_path(conn, :index, ["getting-started"])
defp local_mastodon_root_path(conn) do
case get_session(conn, :return_to) do
nil ->
mastodon_api_path(conn, :index, ["getting-started"])
return_to ->
delete_session(conn, :return_to)
return_to
end
end
defp get_or_make_app do
find_attrs = %{client_name: @local_mastodon_name, redirect_uris: "."}

View File

@ -147,10 +147,18 @@ def render("status.json", %{activity: %{data: %{"object" => object}} = activity}
content =
object
|> render_content()
|> HTML.get_cached_scrubbed_html_for_object(
|> HTML.get_cached_scrubbed_html_for_activity(
User.html_filter_policy(opts[:for]),
activity,
__MODULE__
"mastoapi:content"
)
summary =
(object["summary"] || "")
|> HTML.get_cached_scrubbed_html_for_activity(
User.html_filter_policy(opts[:for]),
activity,
"mastoapi:summary"
)
card = render("card.json", Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity))
@ -182,7 +190,7 @@ def render("status.json", %{activity: %{data: %{"object" => object}} = activity}
muted: CommonAPI.thread_muted?(user, activity) || User.mutes?(opts[:for], user),
pinned: pinned?(activity, user),
sensitive: sensitive,
spoiler_text: object["summary"] || "",
spoiler_text: summary,
visibility: get_visibility(object),
media_attachments: attachments,
mentions: mentions,

View File

@ -12,7 +12,7 @@ def scrub_html_and_truncate(%{data: %{"content" => content}} = object) do
# html content comes from DB already encoded, decode first and scrub after
|> HtmlEntities.decode()
|> String.replace(~r/<br\s?\/?>/, " ")
|> HTML.get_cached_stripped_html_for_object(object, __MODULE__)
|> HTML.get_cached_stripped_html_for_activity(object, "metadata")
|> Formatter.demojify()
|> Formatter.truncate()
end

View File

@ -152,6 +152,7 @@ def token_exchange(
with {_, {:ok, %User{} = user}} <- {:get_user, Authenticator.get_user(conn)},
%App{} = app <- get_app_from_request(conn, params),
{:auth_active, true} <- {:auth_active, User.auth_active?(user)},
{:user_active, true} <- {:user_active, !user.info.deactivated},
scopes <- oauth_scopes(params, app.scopes),
[] <- scopes -- app.scopes,
true <- Enum.any?(scopes),
@ -175,6 +176,11 @@ def token_exchange(
|> put_status(:forbidden)
|> json(%{error: "Your login is missing a confirmed e-mail address"})
{:user_active, false} ->
conn
|> put_status(:forbidden)
|> json(%{error: "Your account is currently disabled"})
_error ->
put_status(conn, 400)
|> json(%{error: "Invalid credentials"})

View File

@ -140,8 +140,12 @@ defmodule Pleroma.Web.Router do
scope "/api/pleroma/admin", Pleroma.Web.AdminAPI do
pipe_through([:admin_api, :oauth_write])
post("/user/follow", AdminAPIController, :user_follow)
post("/user/unfollow", AdminAPIController, :user_unfollow)
get("/users", AdminAPIController, :list_users)
get("/users/:nickname", AdminAPIController, :user_show)
delete("/user", AdminAPIController, :user_delete)
patch("/users/:nickname/toggle_activation", AdminAPIController, :user_toggle_activation)
post("/user", AdminAPIController, :user_create)

View File

@ -254,10 +254,10 @@ def render(
html =
content
|> HTML.get_cached_scrubbed_html_for_object(
|> HTML.get_cached_scrubbed_html_for_activity(
User.html_filter_policy(opts[:for]),
activity,
__MODULE__
"twitterapi:content"
)
|> Formatter.emojify(object["emoji"])
@ -265,7 +265,7 @@ def render(
if content do
content
|> String.replace(~r/<br\s?\/?>/, "\n")
|> HTML.get_cached_stripped_html_for_object(activity, __MODULE__)
|> HTML.get_cached_stripped_html_for_activity(activity, "twitterapi:content")
else
""
end

View File

@ -95,6 +95,13 @@ defp deps do
git: "https://git.pleroma.social/pleroma/auto_linker.git",
ref: "479dd343f4e563ff91215c8275f3b5c67e032850"},
{:pleroma_job_queue, "~> 0.2.0"},
{:telemetry, "~> 0.3"},
{:prometheus_ex, "~> 3.0"},
{:prometheus_plugs, "~> 1.1"},
{:prometheus_phoenix, "~> 1.2"},
{:prometheus_ecto, "~> 1.4"},
{:prometheus_process_collector, "~> 1.4"},
{:recon, github: "ferd/recon", tag: "2.4.0"},
{:quack, "~> 0.1.1"}
]
end

View File

@ -1,4 +1,5 @@
%{
"accept": {:hex, :accept, "0.3.5", "b33b127abca7cc948bbe6caa4c263369abf1347cfa9d8e699c6d214660f10cd1", [:rebar3], [], "hexpm"},
"auto_linker": {:git, "https://git.pleroma.social/pleroma/auto_linker.git", "479dd343f4e563ff91215c8275f3b5c67e032850", [ref: "479dd343f4e563ff91215c8275f3b5c67e032850"]},
"base64url": {:hex, :base64url, "0.0.1", "36a90125f5948e3afd7be97662a1504b934dd5dac78451ca6e9abf85a10286be", [:rebar], [], "hexpm"},
"bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], [], "hexpm"},
@ -57,8 +58,15 @@
"poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm"},
"poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm"},
"postgrex": {:hex, :postgrex, "0.14.1", "63247d4a5ad6b9de57a0bac5d807e1c32d41e39c04b8a4156a26c63bcd8a2e49", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"},
"prometheus": {:hex, :prometheus, "4.2.2", "a830e77b79dc6d28183f4db050a7cac926a6c58f1872f9ef94a35cd989aceef8", [:mix, :rebar3], [], "hexpm"},
"prometheus_ecto": {:hex, :prometheus_ecto, "1.4.1", "6c768ea9654de871e5b32fab2eac348467b3021604ebebbcbd8bcbe806a65ed5", [:mix], [{:ecto, "~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm"},
"prometheus_ex": {:hex, :prometheus_ex, "3.0.5", "fa58cfd983487fc5ead331e9a3e0aa622c67232b3ec71710ced122c4c453a02f", [:mix], [{:prometheus, "~> 4.0", [hex: :prometheus, repo: "hexpm", optional: false]}], "hexpm"},
"prometheus_phoenix": {:hex, :prometheus_phoenix, "1.2.1", "964a74dfbc055f781d3a75631e06ce3816a2913976d1df7830283aa3118a797a", [:mix], [{:phoenix, "~> 1.3", [hex: :phoenix, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.3 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm"},
"prometheus_plugs": {:hex, :prometheus_plugs, "1.1.5", "25933d48f8af3a5941dd7b621c889749894d8a1082a6ff7c67cc99dec26377c5", [:mix], [{:accept, "~> 0.1", [hex: :accept, repo: "hexpm", optional: false]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}, {:prometheus_process_collector, "~> 1.1", [hex: :prometheus_process_collector, repo: "hexpm", optional: true]}], "hexpm"},
"prometheus_process_collector": {:hex, :prometheus_process_collector, "1.4.0", "6dbd39e3165b9ef1c94a7a820e9ffe08479f949dcdd431ed4aaea7b250eebfde", [:rebar3], [{:prometheus, "~> 4.0", [hex: :prometheus, repo: "hexpm", optional: false]}], "hexpm"},
"quack": {:hex, :quack, "0.1.1", "cca7b4da1a233757fdb44b3334fce80c94785b3ad5a602053b7a002b5a8967bf", [:mix], [{:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: false]}, {:tesla, "~> 1.2.0", [hex: :tesla, repo: "hexpm", optional: false]}], "hexpm"},
"ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm"},
"recon": {:git, "https://github.com/ferd/recon.git", "75d70c7c08926d2f24f1ee6de14ee50fe8a52763", [tag: "2.4.0"]},
"ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm"},
"swoosh": {:hex, :swoosh, "0.20.0", "9a6c13822c9815993c03b6f8fccc370fcffb3c158d9754f67b1fdee6b3a5d928", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.12", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug, "~> 1.4", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm"},
"syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]},

View File

@ -47,16 +47,18 @@ test "it authenticates the auth_user if present and password is correct and rese
|> assign(:auth_user, user)
conn =
with_mock User,
reset_password: fn user, %{password: password, password_confirmation: password} ->
send(self(), :reset_password)
{:ok, user}
end do
conn
|> LegacyAuthenticationPlug.call(%{})
with_mocks([
{:crypt, [], [crypt: fn _password, password_hash -> password_hash end]},
{User, [],
[
reset_password: fn user, %{password: password, password_confirmation: password} ->
{:ok, user}
end
]}
]) do
LegacyAuthenticationPlug.call(conn, %{})
end
assert_received :reset_password
assert conn.assigns.user == user
end

View File

@ -240,6 +240,16 @@ def oauth_token_factory do
}
end
def oauth_authorization_factory do
%Pleroma.Web.OAuth.Authorization{
token: :crypto.strong_rand_bytes(32) |> Base.url_encode64(padding: false),
scopes: ["read", "write", "follow", "push"],
valid_until: NaiveDateTime.add(NaiveDateTime.utc_now(), 60 * 10),
user: build(:user),
app: build(:oauth_app)
}
end
def push_subscription_factory do
%Pleroma.Web.Push.Subscription{
user: build(:user),

View File

@ -74,6 +74,52 @@ test "when the user doesn't exist", %{conn: conn} do
end
end
describe "/api/pleroma/admin/user/follow" do
test "allows to force-follow another user" do
admin = insert(:user, info: %{is_admin: true})
user = insert(:user)
follower = insert(:user)
conn =
build_conn()
|> assign(:user, admin)
|> put_req_header("accept", "application/json")
|> post("/api/pleroma/admin/user/follow", %{
"follower" => follower.nickname,
"followed" => user.nickname
})
user = User.get_by_id(user.id)
follower = User.get_by_id(follower.id)
assert User.following?(follower, user)
end
end
describe "/api/pleroma/admin/user/unfollow" do
test "allows to force-unfollow another user" do
admin = insert(:user, info: %{is_admin: true})
user = insert(:user)
follower = insert(:user)
User.follow(follower, user)
conn =
build_conn()
|> assign(:user, admin)
|> put_req_header("accept", "application/json")
|> post("/api/pleroma/admin/user/unfollow", %{
"follower" => follower.nickname,
"followed" => user.nickname
})
user = User.get_by_id(user.id)
follower = User.get_by_id(follower.id)
refute User.following?(follower, user)
end
end
describe "PUT /api/pleroma/admin/users/tag" do
setup do
admin = insert(:user, info: %{is_admin: true})

View File

@ -2340,4 +2340,71 @@ test "accounts fetches correct account for nicknames beginning with numbers", %{
refute acc_one == acc_two
assert acc_two == acc_three
end
describe "index/2 redirections" do
setup %{conn: conn} do
session_opts = [
store: :cookie,
key: "_test",
signing_salt: "cooldude"
]
conn =
conn
|> Plug.Session.call(Plug.Session.init(session_opts))
|> fetch_session()
test_path = "/web/statuses/test"
%{conn: conn, path: test_path}
end
test "redirects not logged-in users to the login page", %{conn: conn, path: path} do
conn = get(conn, path)
assert conn.status == 302
assert redirected_to(conn) == "/web/login"
end
test "does not redirect logged in users to the login page", %{conn: conn, path: path} do
token = insert(:oauth_token)
conn =
conn
|> assign(:user, token.user)
|> put_session(:oauth_token, token.token)
|> get(path)
assert conn.status == 200
end
test "saves referer path to session", %{conn: conn, path: path} do
conn = get(conn, path)
return_to = Plug.Conn.get_session(conn, :return_to)
assert return_to == path
end
test "redirects to the saved path after log in", %{conn: conn, path: path} do
app = insert(:oauth_app, client_name: "Mastodon-Local", redirect_uris: ".")
auth = insert(:oauth_authorization, app: app)
conn =
conn
|> put_session(:return_to, path)
|> get("/web/login", %{code: auth.token})
assert conn.status == 302
assert redirected_to(conn) == path
end
test "redirects to the getting-started page when referer is not present", %{conn: conn} do
app = insert(:oauth_app, client_name: "Mastodon-Local", redirect_uris: ".")
auth = insert(:oauth_authorization, app: app)
conn = get(conn, "/web/login", %{code: auth.token})
assert conn.status == 302
assert redirected_to(conn) == "/web/getting-started"
end
end
end

View File

@ -327,6 +327,32 @@ test "rejects token exchange for valid credentials belonging to unconfirmed user
refute Map.has_key?(resp, "access_token")
end
test "rejects token exchange for valid credentials belonging to deactivated user" do
password = "testpassword"
user =
insert(:user,
password_hash: Comeonin.Pbkdf2.hashpwsalt(password),
info: %{deactivated: true}
)
app = insert(:oauth_app)
conn =
build_conn()
|> post("/oauth/token", %{
"grant_type" => "password",
"username" => user.nickname,
"password" => password,
"client_id" => app.client_id,
"client_secret" => app.client_secret
})
assert resp = json_response(conn, 403)
assert %{"error" => _} = resp
refute Map.has_key?(resp, "access_token")
end
test "rejects an invalid authorization code" do
app = insert(:oauth_app)