Merge branch 'fix/missing_emoji_steal_dir' into 'develop'

Create dir for EmojiStealPolicy automatically

See merge request pleroma/pleroma!3218
This commit is contained in:
lain 2021-01-08 13:10:49 +00:00 committed by rinpatch
parent 197606984d
commit 3dc04b9aca
3 changed files with 122 additions and 76 deletions

View File

@ -3,6 +3,12 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
## Unreleased (Patch)
### Fixed
- StealEmojiPolicy creates dir for emojis, if it doesn't exist.
## [2.2.1] - 2020-12-22 ## [2.2.1] - 2020-12-22
### Changed ### Changed
@ -17,6 +23,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Rich Media Previews sometimes showed the wrong preview due to a bug following redirects. - Rich Media Previews sometimes showed the wrong preview due to a bug following redirects.
- Fixes for the autolinker. - Fixes for the autolinker.
- Forwarded reports duplication from Pleroma instances. - Forwarded reports duplication from Pleroma instances.
- Emoji Reaction activity filtering from blocked and muted accounts.
- <details> - <details>
<summary>API</summary> <summary>API</summary>

View File

@ -10,73 +10,75 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do
@moduledoc "Detect new emojis by their shortcode and steals them" @moduledoc "Detect new emojis by their shortcode and steals them"
@behaviour Pleroma.Web.ActivityPub.MRF @behaviour Pleroma.Web.ActivityPub.MRF
defp remote_host?(host), do: host != Config.get([Pleroma.Web.Endpoint, :url, :host])
defp accept_host?(host), do: host in Config.get([:mrf_steal_emoji, :hosts], []) defp accept_host?(host), do: host in Config.get([:mrf_steal_emoji, :hosts], [])
defp steal_emoji({shortcode, url}) do defp steal_emoji({shortcode, url}, emoji_dir_path) do
url = Pleroma.Web.MediaProxy.url(url) url = Pleroma.Web.MediaProxy.url(url)
{:ok, response} = Pleroma.HTTP.get(url)
size_limit = Config.get([:mrf_steal_emoji, :size_limit], 50_000)
if byte_size(response.body) <= size_limit do with {:ok, %{status: status} = response} when status in 200..299 <- Pleroma.HTTP.get(url) do
emoji_dir_path = size_limit = Config.get([:mrf_steal_emoji, :size_limit], 50_000)
Config.get(
[:mrf_steal_emoji, :path], if byte_size(response.body) <= size_limit do
Path.join(Config.get([:instance, :static_dir]), "emoji/stolen") extension =
url
|> URI.parse()
|> Map.get(:path)
|> Path.basename()
|> Path.extname()
file_path = Path.join(emoji_dir_path, shortcode <> (extension || ".png"))
case File.write(file_path, response.body) do
:ok ->
shortcode
e ->
Logger.warn("MRF.StealEmojiPolicy: Failed to write to #{file_path}: #{inspect(e)}")
nil
end
else
Logger.debug(
"MRF.StealEmojiPolicy: :#{shortcode}: at #{url} (#{byte_size(response.body)} B) over size limit (#{
size_limit
} B)"
) )
extension = nil
url
|> URI.parse()
|> Map.get(:path)
|> Path.basename()
|> Path.extname()
file_path = Path.join([emoji_dir_path, shortcode <> (extension || ".png")])
try do
:ok = File.write(file_path, response.body)
shortcode
rescue
e ->
Logger.warn("MRF.StealEmojiPolicy: Failed to write to #{file_path}: #{inspect(e)}")
nil
end end
else else
Logger.debug( e ->
"MRF.StealEmojiPolicy: :#{shortcode}: at #{url} (#{byte_size(response.body)} B) over size limit (#{ Logger.warn("MRF.StealEmojiPolicy: Failed to fetch #{url}: #{inspect(e)}")
size_limit nil
} B)"
)
nil
end end
rescue
e ->
Logger.warn("MRF.StealEmojiPolicy: Failed to fetch #{url}: #{inspect(e)}")
nil
end end
@impl true @impl true
def filter(%{"object" => %{"emoji" => foreign_emojis, "actor" => actor}} = message) do def filter(%{"object" => %{"emoji" => foreign_emojis, "actor" => actor}} = message) do
host = URI.parse(actor).host host = URI.parse(actor).host
if remote_host?(host) and accept_host?(host) do if host != Pleroma.Web.Endpoint.host() and accept_host?(host) do
installed_emoji = Pleroma.Emoji.get_all() |> Enum.map(fn {k, _} -> k end) installed_emoji = Pleroma.Emoji.get_all() |> Enum.map(fn {k, _} -> k end)
emoji_dir_path =
Config.get(
[:mrf_steal_emoji, :path],
Path.join(Config.get([:instance, :static_dir]), "emoji/stolen")
)
File.mkdir_p(emoji_dir_path)
new_emojis = new_emojis =
foreign_emojis foreign_emojis
|> Enum.filter(fn {shortcode, _url} -> shortcode not in installed_emoji end) |> Enum.reject(fn {shortcode, _url} -> shortcode in installed_emoji end)
|> Enum.filter(fn {shortcode, _url} -> |> Enum.filter(fn {shortcode, _url} ->
reject_emoji? = reject_emoji? =
Config.get([:mrf_steal_emoji, :rejected_shortcodes], []) [:mrf_steal_emoji, :rejected_shortcodes]
|> Config.get([])
|> Enum.find(false, fn regex -> String.match?(shortcode, regex) end) |> Enum.find(false, fn regex -> String.match?(shortcode, regex) end)
!reject_emoji? !reject_emoji?
end) end)
|> Enum.map(&steal_emoji(&1)) |> Enum.map(&steal_emoji(&1, emoji_dir_path))
|> Enum.filter(& &1) |> Enum.filter(& &1)
if !Enum.empty?(new_emojis) do if !Enum.empty?(new_emojis) do

View File

@ -6,6 +6,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicyTest do
use Pleroma.DataCase use Pleroma.DataCase
alias Pleroma.Config alias Pleroma.Config
alias Pleroma.Emoji
alias Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy alias Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy
setup_all do setup_all do
@ -14,55 +15,91 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicyTest do
end end
setup do setup do
emoji_path = Path.join(Config.get([:instance, :static_dir]), "emoji/stolen") emoji_path = [:instance, :static_dir] |> Config.get() |> Path.join("emoji/stolen")
File.rm_rf!(emoji_path)
File.mkdir!(emoji_path)
Pleroma.Emoji.reload() Emoji.reload()
message = %{
"type" => "Create",
"object" => %{
"emoji" => [{"firedfox", "https://example.org/emoji/firedfox.png"}],
"actor" => "https://example.org/users/admin"
}
}
on_exit(fn -> on_exit(fn ->
File.rm_rf!(emoji_path) File.rm_rf!(emoji_path)
end) end)
:ok [message: message, path: emoji_path]
end end
test "does nothing by default" do test "does nothing by default", %{message: message} do
installed_emoji = Pleroma.Emoji.get_all() |> Enum.map(fn {k, _} -> k end) refute "firedfox" in installed()
refute "firedfox" in installed_emoji
message = %{ assert {:ok, _message} = StealEmojiPolicy.filter(message)
"type" => "Create",
"object" => %{
"emoji" => [{"firedfox", "https://example.org/emoji/firedfox.png"}],
"actor" => "https://example.org/users/admin"
}
}
assert {:ok, message} == StealEmojiPolicy.filter(message) refute "firedfox" in installed()
installed_emoji = Pleroma.Emoji.get_all() |> Enum.map(fn {k, _} -> k end)
refute "firedfox" in installed_emoji
end end
test "Steals emoji on unknown shortcode from allowed remote host" do test "Steals emoji on unknown shortcode from allowed remote host", %{
installed_emoji = Pleroma.Emoji.get_all() |> Enum.map(fn {k, _} -> k end) message: message,
refute "firedfox" in installed_emoji path: path
} do
refute "firedfox" in installed()
refute File.exists?(path)
message = %{ clear_config(:mrf_steal_emoji, hosts: ["example.org"], size_limit: 284_468)
"type" => "Create",
"object" => %{
"emoji" => [{"firedfox", "https://example.org/emoji/firedfox.png"}],
"actor" => "https://example.org/users/admin"
}
}
clear_config([:mrf_steal_emoji, :hosts], ["example.org"]) assert {:ok, _message} = StealEmojiPolicy.filter(message)
clear_config([:mrf_steal_emoji, :size_limit], 284_468)
assert {:ok, message} == StealEmojiPolicy.filter(message) assert "firedfox" in installed()
assert File.exists?(path)
installed_emoji = Pleroma.Emoji.get_all() |> Enum.map(fn {k, _} -> k end) assert path
assert "firedfox" in installed_emoji |> Path.join("firedfox.png")
|> File.exists?()
end end
test "reject shortcode", %{message: message} do
refute "firedfox" in installed()
clear_config(:mrf_steal_emoji,
hosts: ["example.org"],
size_limit: 284_468,
rejected_shortcodes: [~r/firedfox/]
)
assert {:ok, _message} = StealEmojiPolicy.filter(message)
refute "firedfox" in installed()
end
test "reject if size is above the limit", %{message: message} do
refute "firedfox" in installed()
clear_config(:mrf_steal_emoji, hosts: ["example.org"], size_limit: 50_000)
assert {:ok, _message} = StealEmojiPolicy.filter(message)
refute "firedfox" in installed()
end
test "reject if host returns error", %{message: message} do
refute "firedfox" in installed()
Tesla.Mock.mock(fn %{method: :get, url: "https://example.org/emoji/firedfox.png"} ->
{:ok, %Tesla.Env{status: 404, body: "Not found"}}
end)
clear_config(:mrf_steal_emoji, hosts: ["example.org"], size_limit: 284_468)
ExUnit.CaptureLog.capture_log(fn ->
assert {:ok, _message} = StealEmojiPolicy.filter(message)
end) =~ "MRF.StealEmojiPolicy: Failed to fetch https://example.org/emoji/firedfox.png"
refute "firedfox" in installed()
end
defp installed, do: Emoji.get_all() |> Enum.map(fn {k, _} -> k end)
end end