From 5b1eeb06d81872696fac89dba457fe62b62d6182 Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 21 Jul 2020 22:18:17 +0000 Subject: [PATCH 1/6] Revert "Merge branch 'revert-2b5d9eb1' into 'develop'" This reverts merge request !2784 --- CHANGELOG.md | 1 + config/config.exs | 18 ++++----- config/description.exs | 20 +++++++--- docs/configuration/cheatsheet.md | 35 +++++++++--------- lib/pleroma/config/config_db.ex | 1 - lib/pleroma/formatter.ex | 26 +++++++------ lib/pleroma/web/rich_media/helpers.ex | 4 +- mix.exs | 4 +- mix.lock | 2 +- .../20200716195806_autolinker_to_linkify.exs | 37 +++++++++++++++++++ test/formatter_test.exs | 30 +++++++++++++++ 11 files changed, 126 insertions(+), 52 deletions(-) create mode 100644 priv/repo/migrations/20200716195806_autolinker_to_linkify.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index 080270073..f4397ec3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Changed - **Breaking:** Elixir >=1.9 is now required (was >= 1.8) +- **Breaking:** Configuration: `:auto_linker, :opts` moved to `:pleroma, Pleroma.Formatter`. Old config namespace is deprecated. - In Conversations, return only direct messages as `last_status` - Using the `only_media` filter on timelines will now exclude reblog media - MFR policy to set global expiration for all local Create activities diff --git a/config/config.exs b/config/config.exs index 2d3f35e70..406bf2a9b 100644 --- a/config/config.exs +++ b/config/config.exs @@ -527,16 +527,14 @@ federator_outgoing: 5 ] -config :auto_linker, - opts: [ - extra: true, - # TODO: Set to :no_scheme when it works properly - validate_tld: true, - class: false, - strip_prefix: false, - new_window: false, - rel: "ugc" - ] +config :pleroma, Pleroma.Formatter, + class: false, + rel: "ugc", + new_window: false, + truncate: false, + strip_prefix: false, + extra: true, + validate_tld: :no_scheme config :pleroma, :ldap, enabled: System.get_env("LDAP_ENABLED") == "true", diff --git a/config/description.exs b/config/description.exs index f1c6773f1..b97b0a7ec 100644 --- a/config/description.exs +++ b/config/description.exs @@ -2216,11 +2216,12 @@ ] }, %{ - group: :auto_linker, - key: :opts, + group: :pleroma, + key: Pleroma.Formatter, label: "Auto Linker", type: :group, - description: "Configuration for the auto_linker library", + description: + "Configuration for Pleroma's link formatter which parses mentions, hashtags, and URLs.", children: [ %{ key: :class, @@ -2237,24 +2238,31 @@ %{ key: :new_window, type: :boolean, - description: "Link URLs will open in new window/tab" + description: "Link URLs will open in a new window/tab." }, %{ key: :truncate, type: [:integer, false], description: - "Set to a number to truncate URLs longer then the number. Truncated URLs will end in `..`", + "Set to a number to truncate URLs longer than the number. Truncated URLs will end in `...`", suggestions: [15, false] }, %{ key: :strip_prefix, type: :boolean, - description: "Strip the scheme prefix" + description: "Strip the scheme prefix." }, %{ key: :extra, type: :boolean, description: "Link URLs with rarely used schemes (magnet, ipfs, irc, etc.)" + }, + %{ + key: :validate_tld, + type: [:atom, :boolean], + description: + "Set to false to disable TLD validation for URLs/emails. Can be set to :no_scheme to validate TLDs only for URLs without a scheme (e.g `example.com` will be validated, but `http://example.loki` won't)", + suggestions: [:no_scheme, true] } ] }, diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 6c1babba3..042ad30c9 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -934,30 +934,29 @@ Configure OAuth 2 provider capabilities: ### :uri_schemes * `valid_schemes`: List of the scheme part that is considered valid to be an URL. -### :auto_linker +### Pleroma.Formatter -Configuration for the `auto_linker` library: +Configuration for Pleroma's link formatter which parses mentions, hashtags, and URLs. -* `class: "auto-linker"` - specify the class to be added to the generated link. false to clear. -* `rel: "noopener noreferrer"` - override the rel attribute. false to clear. -* `new_window: true` - set to false to remove `target='_blank'` attribute. -* `scheme: false` - Set to true to link urls with schema `http://google.com`. -* `truncate: false` - Set to a number to truncate urls longer then the number. Truncated urls will end in `..`. -* `strip_prefix: true` - Strip the scheme prefix. -* `extra: false` - link urls with rarely used schemes (magnet, ipfs, irc, etc.). +* `class` - specify the class to be added to the generated link (default: `false`) +* `rel` - specify the rel attribute (default: `ugc`) +* `new_window` - adds `target="_blank"` attribute (default: `false`) +* `truncate` - Set to a number to truncate URLs longer then the number. Truncated URLs will end in `...` (default: `false`) +* `strip_prefix` - Strip the scheme prefix (default: `false`) +* `extra` - link URLs with rarely used schemes (magnet, ipfs, irc, etc.) (default: `true`) +* `validate_tld` - Set to false to disable TLD validation for URLs/emails. Can be set to :no_scheme to validate TLDs only for urls without a scheme (e.g `example.com` will be validated, but `http://example.loki` won't) (default: `:no_scheme`) Example: ```elixir -config :auto_linker, - opts: [ - scheme: true, - extra: true, - class: false, - strip_prefix: false, - new_window: false, - rel: "ugc" - ] +config :pleroma, Pleroma.Formatter, + class: false, + rel: "ugc", + new_window: false, + truncate: false, + strip_prefix: false, + extra: true, + validate_tld: :no_scheme ``` ## Custom Runtime Modules (`:modules`) diff --git a/lib/pleroma/config/config_db.ex b/lib/pleroma/config/config_db.ex index 1a89d8895..e5b7811aa 100644 --- a/lib/pleroma/config/config_db.ex +++ b/lib/pleroma/config/config_db.ex @@ -156,7 +156,6 @@ defp only_full_update?(%ConfigDB{group: group, key: key}) do {:quack, :meta}, {:mime, :types}, {:cors_plug, [:max_age, :methods, :expose, :headers]}, - {:auto_linker, :opts}, {:swarm, :node_blacklist}, {:logger, :backends} ] diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index 02a93a8dc..0c450eae4 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -10,11 +10,15 @@ defmodule Pleroma.Formatter do @link_regex ~r"((?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~%:/?#[\]@!\$&'\(\)\*\+,;=.]+)|[0-9a-z+\-\.]+:[0-9a-z$-_.+!*'(),]+"ui @markdown_characters_regex ~r/(`|\*|_|{|}|[|]|\(|\)|#|\+|-|\.|!)/ - @auto_linker_config hashtag: true, - hashtag_handler: &Pleroma.Formatter.hashtag_handler/4, - mention: true, - mention_handler: &Pleroma.Formatter.mention_handler/4, - scheme: true + defp linkify_opts do + Pleroma.Config.get(Pleroma.Formatter) ++ + [ + hashtag: true, + hashtag_handler: &Pleroma.Formatter.hashtag_handler/4, + mention: true, + mention_handler: &Pleroma.Formatter.mention_handler/4 + ] + end def escape_mention_handler("@" <> nickname = mention, buffer, _, _) do case User.get_cached_by_nickname(nickname) do @@ -80,19 +84,19 @@ def hashtag_handler("#" <> tag = tag_text, _buffer, _opts, acc) do @spec linkify(String.t(), keyword()) :: {String.t(), [{String.t(), User.t()}], [{String.t(), String.t()}]} def linkify(text, options \\ []) do - options = options ++ @auto_linker_config + options = linkify_opts() ++ options if options[:safe_mention] && Regex.named_captures(@safe_mention_regex, text) do %{"mentions" => mentions, "rest" => rest} = Regex.named_captures(@safe_mention_regex, text) acc = %{mentions: MapSet.new(), tags: MapSet.new()} - {text_mentions, %{mentions: mentions}} = AutoLinker.link_map(mentions, acc, options) - {text_rest, %{tags: tags}} = AutoLinker.link_map(rest, acc, options) + {text_mentions, %{mentions: mentions}} = Linkify.link_map(mentions, acc, options) + {text_rest, %{tags: tags}} = Linkify.link_map(rest, acc, options) {text_mentions <> text_rest, MapSet.to_list(mentions), MapSet.to_list(tags)} else acc = %{mentions: MapSet.new(), tags: MapSet.new()} - {text, %{mentions: mentions, tags: tags}} = AutoLinker.link_map(text, acc, options) + {text, %{mentions: mentions, tags: tags}} = Linkify.link_map(text, acc, options) {text, MapSet.to_list(mentions), MapSet.to_list(tags)} end @@ -111,9 +115,9 @@ def mentions_escape(text, options \\ []) do if options[:safe_mention] && Regex.named_captures(@safe_mention_regex, text) do %{"mentions" => mentions, "rest" => rest} = Regex.named_captures(@safe_mention_regex, text) - AutoLinker.link(mentions, options) <> AutoLinker.link(rest, options) + Linkify.link(mentions, options) <> Linkify.link(rest, options) else - AutoLinker.link(text, options) + Linkify.link(text, options) end end diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index 1729141e9..747f2dc6b 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -11,10 +11,10 @@ defmodule Pleroma.Web.RichMedia.Helpers do @spec validate_page_url(URI.t() | binary()) :: :ok | :error defp validate_page_url(page_url) when is_binary(page_url) do - validate_tld = Application.get_env(:auto_linker, :opts)[:validate_tld] + validate_tld = Pleroma.Config.get([Pleroma.Formatter, :validate_tld]) page_url - |> AutoLinker.Parser.url?(scheme: true, validate_tld: validate_tld) + |> Linkify.Parser.url?(validate_tld: validate_tld) |> parse_uri(page_url) end diff --git a/mix.exs b/mix.exs index 52b4cf268..f44d7a887 100644 --- a/mix.exs +++ b/mix.exs @@ -166,9 +166,7 @@ defp deps do {:floki, "~> 0.25"}, {:timex, "~> 3.5"}, {:ueberauth, "~> 0.4"}, - {:auto_linker, - git: "https://git.pleroma.social/pleroma/auto_linker.git", - ref: "95e8188490e97505c56636c1379ffdf036c1fdde"}, + {:linkify, "~> 0.2.0"}, {:http_signatures, git: "https://git.pleroma.social/pleroma/http_signatures.git", ref: "293d77bb6f4a67ac8bde1428735c3b42f22cbb30"}, diff --git a/mix.lock b/mix.lock index 8dd37a40f..6430ddd19 100644 --- a/mix.lock +++ b/mix.lock @@ -1,6 +1,5 @@ %{ "accept": {:hex, :accept, "0.3.5", "b33b127abca7cc948bbe6caa4c263369abf1347cfa9d8e699c6d214660f10cd1", [:rebar3], [], "hexpm", "11b18c220bcc2eab63b5470c038ef10eb6783bcb1fcdb11aa4137defa5ac1bb8"}, - "auto_linker": {:git, "https://git.pleroma.social/pleroma/auto_linker.git", "95e8188490e97505c56636c1379ffdf036c1fdde", [ref: "95e8188490e97505c56636c1379ffdf036c1fdde"]}, "base62": {:hex, :base62, "1.2.1", "4866763e08555a7b3917064e9eef9194c41667276c51b59de2bc42c6ea65f806", [:mix], [{:custom_base, "~> 0.2.1", [hex: :custom_base, repo: "hexpm", optional: false]}], "hexpm", "3b29948de2013d3f93aa898c884a9dff847e7aec75d9d6d8c1dc4c61c2716c42"}, "base64url": {:hex, :base64url, "0.0.1", "36a90125f5948e3afd7be97662a1504b934dd5dac78451ca6e9abf85a10286be", [:rebar], [], "hexpm"}, "bbcode": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/bbcode.git", "f2d267675e9a7e1ad1ea9beb4cc23382762b66c2", [ref: "v0.2.0"]}, @@ -63,6 +62,7 @@ "jose": {:hex, :jose, "1.10.1", "16d8e460dae7203c6d1efa3f277e25b5af8b659febfc2f2eb4bacf87f128b80a", [:mix, :rebar3], [], "hexpm", "3c7ddc8a9394b92891db7c2771da94bf819834a1a4c92e30857b7d582e2f8257"}, "jumper": {:hex, :jumper, "1.0.1", "3c00542ef1a83532b72269fab9f0f0c82bf23a35e27d278bfd9ed0865cecabff", [:mix], [], "hexpm", "318c59078ac220e966d27af3646026db9b5a5e6703cb2aa3e26bcfaba65b7433"}, "libring": {:hex, :libring, "1.4.0", "41246ba2f3fbc76b3971f6bce83119dfec1eee17e977a48d8a9cfaaf58c2a8d6", [:mix], [], "hexpm"}, + "linkify": {:hex, :linkify, "0.2.0", "2518bbbea21d2caa9d372424e1ad845b640c6630e2d016f1bd1f518f9ebcca28", [:mix], [], "hexpm", "b8ca8a68b79e30b7938d6c996085f3db14939f29538a59ca5101988bb7f917f6"}, "makeup": {:hex, :makeup, "1.0.0", "671df94cf5a594b739ce03b0d0316aa64312cee2574b6a44becb83cd90fb05dc", [:mix], [{:nimble_parsec, "~> 0.5.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "a10c6eb62cca416019663129699769f0c2ccf39428b3bb3c0cb38c718a0c186d"}, "makeup_elixir": {:hex, :makeup_elixir, "0.14.0", "cf8b7c66ad1cff4c14679698d532f0b5d45a3968ffbcbfd590339cb57742f1ae", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "d4b316c7222a85bbaa2fd7c6e90e37e953257ad196dc229505137c5e505e9eff"}, "meck": {:hex, :meck, "0.8.13", "ffedb39f99b0b99703b8601c6f17c7f76313ee12de6b646e671e3188401f7866", [:rebar3], [], "hexpm", "d34f013c156db51ad57cc556891b9720e6a1c1df5fe2e15af999c84d6cebeb1a"}, diff --git a/priv/repo/migrations/20200716195806_autolinker_to_linkify.exs b/priv/repo/migrations/20200716195806_autolinker_to_linkify.exs new file mode 100644 index 000000000..9ec4203eb --- /dev/null +++ b/priv/repo/migrations/20200716195806_autolinker_to_linkify.exs @@ -0,0 +1,37 @@ +defmodule Pleroma.Repo.Migrations.AutolinkerToLinkify do + use Ecto.Migration + + alias Pleroma.Repo + alias Pleroma.ConfigDB + + @autolinker_path %{group: :auto_linker, key: :opts} + @linkify_path %{group: :pleroma, key: Pleroma.Formatter} + + @compat_opts [:class, :rel, :new_window, :truncate, :strip_prefix, :extra] + + def change do + with {:ok, {old, new}} <- maybe_get_params() do + move_config(old, new) + end + end + + defp move_config(%{} = old, %{} = new) do + {:ok, _} = ConfigDB.update_or_create(new) + {:ok, _} = ConfigDB.delete(old) + :ok + end + + defp maybe_get_params() do + with %ConfigDB{value: opts} <- ConfigDB.get_by_params(@autolinker_path), + %{} = opts <- transform_opts(opts), + %{} = linkify_params <- Map.put(@linkify_path, :value, opts) do + {:ok, {@autolinker_path, linkify_params}} + end + end + + defp transform_opts(opts) when is_list(opts) do + opts + |> Enum.into(%{}) + |> Map.take(@compat_opts) + end +end diff --git a/test/formatter_test.exs b/test/formatter_test.exs index bef5a2c28..8713ab9c2 100644 --- a/test/formatter_test.exs +++ b/test/formatter_test.exs @@ -255,6 +255,36 @@ test "it can parse mentions and return the relevant users" do assert {_text, ^expected_mentions, []} = Formatter.linkify(text) end + + test "it parses URL containing local mention" do + _user = insert(:user, %{nickname: "lain"}) + + text = "https://example.com/@lain" + + expected = ~S(https://example.com/@lain) + + assert {^expected, [], []} = Formatter.linkify(text) + end + + test "it correctly parses angry face D:< with mention" do + lain = + insert(:user, %{ + nickname: "lain@lain.com", + ap_id: "https://lain.com/users/lain", + id: "9qrWmR0cKniB0YU0TA" + }) + + text = "@lain@lain.com D:<" + + expected_text = + ~S(@lain D:<) + + expected_mentions = [ + {"@lain@lain.com", lain} + ] + + assert {^expected_text, ^expected_mentions, []} = Formatter.linkify(text) + end end describe ".parse_tags" do From 7045db5a506aa672d141dc33cfadd53208b4d067 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 22 Jul 2020 11:27:52 -0500 Subject: [PATCH 2/6] Fix linkify ConfigDB migration --- priv/repo/migrations/20200716195806_autolinker_to_linkify.exs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/priv/repo/migrations/20200716195806_autolinker_to_linkify.exs b/priv/repo/migrations/20200716195806_autolinker_to_linkify.exs index 9ec4203eb..782a3cc55 100644 --- a/priv/repo/migrations/20200716195806_autolinker_to_linkify.exs +++ b/priv/repo/migrations/20200716195806_autolinker_to_linkify.exs @@ -23,7 +23,7 @@ defp move_config(%{} = old, %{} = new) do defp maybe_get_params() do with %ConfigDB{value: opts} <- ConfigDB.get_by_params(@autolinker_path), - %{} = opts <- transform_opts(opts), + opts <- transform_opts(opts), %{} = linkify_params <- Map.put(@linkify_path, :value, opts) do {:ok, {@autolinker_path, linkify_params}} end @@ -33,5 +33,6 @@ defp transform_opts(opts) when is_list(opts) do opts |> Enum.into(%{}) |> Map.take(@compat_opts) + |> Map.to_list() end end From 67389b77af7c6f9ccd18ec385b6ef4fd102e3eb6 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 22 Jul 2020 13:10:10 -0500 Subject: [PATCH 3/6] Add AutolinkerToLinkify migration test --- .../20200716195806_autolinker_to_linkify.exs | 4 +- ...00716195806_autolinker_to_linkify_test.exs | 68 +++++++++++++++++++ 2 files changed, 69 insertions(+), 3 deletions(-) create mode 100644 test/migrations/20200716195806_autolinker_to_linkify_test.exs diff --git a/priv/repo/migrations/20200716195806_autolinker_to_linkify.exs b/priv/repo/migrations/20200716195806_autolinker_to_linkify.exs index 782a3cc55..570acba84 100644 --- a/priv/repo/migrations/20200716195806_autolinker_to_linkify.exs +++ b/priv/repo/migrations/20200716195806_autolinker_to_linkify.exs @@ -1,7 +1,5 @@ defmodule Pleroma.Repo.Migrations.AutolinkerToLinkify do use Ecto.Migration - - alias Pleroma.Repo alias Pleroma.ConfigDB @autolinker_path %{group: :auto_linker, key: :opts} @@ -29,7 +27,7 @@ defp maybe_get_params() do end end - defp transform_opts(opts) when is_list(opts) do + def transform_opts(opts) when is_list(opts) do opts |> Enum.into(%{}) |> Map.take(@compat_opts) diff --git a/test/migrations/20200716195806_autolinker_to_linkify_test.exs b/test/migrations/20200716195806_autolinker_to_linkify_test.exs new file mode 100644 index 000000000..362cf5535 --- /dev/null +++ b/test/migrations/20200716195806_autolinker_to_linkify_test.exs @@ -0,0 +1,68 @@ +defmodule Pleroma.Repo.Migrations.AutolinkerToLinkifyTest do + use Pleroma.DataCase + import Pleroma.Factory + alias Pleroma.ConfigDB + + setup_all do + [{module, _}] = + Code.require_file("20200716195806_autolinker_to_linkify.exs", "priv/repo/migrations") + + {:ok, %{migration: module}} + end + + test "change/0 converts auto_linker opts for Pleroma.Formatter", %{migration: migration} do + autolinker_opts = [ + extra: true, + validate_tld: true, + class: false, + strip_prefix: false, + new_window: false, + rel: "ugc" + ] + + insert(:config, group: :auto_linker, key: :opts, value: autolinker_opts) + + migration.change() + + assert nil == ConfigDB.get_by_params(%{group: :auto_linker, key: :opts}) + + %{value: new_opts} = ConfigDB.get_by_params(%{group: :pleroma, key: Pleroma.Formatter}) + + assert new_opts == [ + class: false, + extra: true, + new_window: false, + rel: "ugc", + strip_prefix: false + ] + + {text, _mentions, []} = + Pleroma.Formatter.linkify( + "https://www.businessinsider.com/walmart-will-close-stores-on-thanksgiving-ending-black-friday-tradition-2020-7\n\nOmg will COVID finally end Black Friday???" + ) + + assert text == + "https://www.businessinsider.com/walmart-will-close-stores-on-thanksgiving-ending-black-friday-tradition-2020-7\n\nOmg will COVID finally end Black Friday???" + end + + test "transform_opts/1 returns a list of compatible opts", %{migration: migration} do + old_opts = [ + extra: true, + validate_tld: true, + class: false, + strip_prefix: false, + new_window: false, + rel: "ugc" + ] + + expected_opts = [ + class: false, + extra: true, + new_window: false, + rel: "ugc", + strip_prefix: false + ] + + assert migration.transform_opts(old_opts) == expected_opts + end +end From b87a1f8eaff7e5663fd4b84b43be350754eb37d2 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 22 Jul 2020 13:45:15 -0500 Subject: [PATCH 4/6] Refactor require_migration/1 into a test helper function --- .../20200716195806_autolinker_to_linkify_test.exs | 8 ++------ test/support/helpers.ex | 5 +++++ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/test/migrations/20200716195806_autolinker_to_linkify_test.exs b/test/migrations/20200716195806_autolinker_to_linkify_test.exs index 362cf5535..063dab0f7 100644 --- a/test/migrations/20200716195806_autolinker_to_linkify_test.exs +++ b/test/migrations/20200716195806_autolinker_to_linkify_test.exs @@ -1,14 +1,10 @@ defmodule Pleroma.Repo.Migrations.AutolinkerToLinkifyTest do use Pleroma.DataCase import Pleroma.Factory + import Pleroma.Tests.Helpers, only: [require_migration: 1] alias Pleroma.ConfigDB - setup_all do - [{module, _}] = - Code.require_file("20200716195806_autolinker_to_linkify.exs", "priv/repo/migrations") - - {:ok, %{migration: module}} - end + setup_all do: require_migration("20200716195806_autolinker_to_linkify") test "change/0 converts auto_linker opts for Pleroma.Formatter", %{migration: migration} do autolinker_opts = [ diff --git a/test/support/helpers.ex b/test/support/helpers.ex index 26281b45e..5cbf2e291 100644 --- a/test/support/helpers.ex +++ b/test/support/helpers.ex @@ -32,6 +32,11 @@ defmacro clear_config(config_path, temp_setting) do end end + def require_migration(migration_name) do + [{module, _}] = Code.require_file("#{migration_name}.exs", "priv/repo/migrations") + {:ok, %{migration: module}} + end + defmacro __using__(_opts) do quote do import Pleroma.Tests.Helpers, From c7a0016f9f4731c58a7989c7ee10e19d3f90d2eb Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 22 Jul 2020 14:18:09 -0500 Subject: [PATCH 5/6] Migration to fix malformed Pleroma.Formatter config --- ...2185515_fix_malformed_formatter_config.exs | 26 ++++++++ ...15_fix_malformed_formatter_config_test.exs | 62 +++++++++++++++++++ 2 files changed, 88 insertions(+) create mode 100644 priv/repo/migrations/20200722185515_fix_malformed_formatter_config.exs create mode 100644 test/migrations/20200722185515_fix_malformed_formatter_config_test.exs diff --git a/priv/repo/migrations/20200722185515_fix_malformed_formatter_config.exs b/priv/repo/migrations/20200722185515_fix_malformed_formatter_config.exs new file mode 100644 index 000000000..77b760825 --- /dev/null +++ b/priv/repo/migrations/20200722185515_fix_malformed_formatter_config.exs @@ -0,0 +1,26 @@ +defmodule Pleroma.Repo.Migrations.FixMalformedFormatterConfig do + use Ecto.Migration + alias Pleroma.ConfigDB + + @config_path %{group: :pleroma, key: Pleroma.Formatter} + + def change do + with %ConfigDB{value: %{} = opts} <- ConfigDB.get_by_params(@config_path), + fixed_opts <- Map.to_list(opts) do + fix_config(fixed_opts) + else + _ -> :skipped + end + end + + defp fix_config(fixed_opts) when is_list(fixed_opts) do + {:ok, _} = + ConfigDB.update_or_create(%{ + group: :pleroma, + key: Pleroma.Formatter, + value: fixed_opts + }) + + :ok + end +end diff --git a/test/migrations/20200722185515_fix_malformed_formatter_config_test.exs b/test/migrations/20200722185515_fix_malformed_formatter_config_test.exs new file mode 100644 index 000000000..9e8f997a0 --- /dev/null +++ b/test/migrations/20200722185515_fix_malformed_formatter_config_test.exs @@ -0,0 +1,62 @@ +defmodule Pleroma.Repo.Migrations.FixMalformedFormatterConfigTest do + use Pleroma.DataCase + import Pleroma.Factory + import Pleroma.Tests.Helpers, only: [require_migration: 1] + alias Pleroma.ConfigDB + + setup_all do: require_migration("20200722185515_fix_malformed_formatter_config") + + test "change/0 converts a map into a list", %{migration: migration} do + incorrect_opts = %{ + class: false, + extra: true, + new_window: false, + rel: "ugc", + strip_prefix: false + } + + insert(:config, group: :pleroma, key: Pleroma.Formatter, value: incorrect_opts) + + assert :ok == migration.change() + + %{value: new_opts} = ConfigDB.get_by_params(%{group: :pleroma, key: Pleroma.Formatter}) + + assert new_opts == [ + class: false, + extra: true, + new_window: false, + rel: "ugc", + strip_prefix: false + ] + + {text, _mentions, []} = + Pleroma.Formatter.linkify( + "https://www.businessinsider.com/walmart-will-close-stores-on-thanksgiving-ending-black-friday-tradition-2020-7\n\nOmg will COVID finally end Black Friday???" + ) + + assert text == + "https://www.businessinsider.com/walmart-will-close-stores-on-thanksgiving-ending-black-friday-tradition-2020-7\n\nOmg will COVID finally end Black Friday???" + end + + test "change/0 skips if Pleroma.Formatter config is already a list", %{migration: migration} do + opts = [ + class: false, + extra: true, + new_window: false, + rel: "ugc", + strip_prefix: false + ] + + insert(:config, group: :pleroma, key: Pleroma.Formatter, value: opts) + + assert :skipped == migration.change() + + %{value: new_opts} = ConfigDB.get_by_params(%{group: :pleroma, key: Pleroma.Formatter}) + + assert new_opts == opts + end + + test "change/0 skips if Pleroma.Formatter is empty", %{migration: migration} do + assert :skipped == migration.change() + end +end From b6488a4db4accc6cda716c5fdfb03f5a30ddf3d4 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 22 Jul 2020 16:01:55 -0500 Subject: [PATCH 6/6] Update linkify migration tests to use config from ConfigDB --- test/formatter_test.exs | 1 + ...20200716195806_autolinker_to_linkify_test.exs | 16 ++++++++++------ ...85515_fix_malformed_formatter_config_test.exs | 12 ++++++++---- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/test/formatter_test.exs b/test/formatter_test.exs index 8713ab9c2..f066bd50a 100644 --- a/test/formatter_test.exs +++ b/test/formatter_test.exs @@ -10,6 +10,7 @@ defmodule Pleroma.FormatterTest do import Pleroma.Factory setup_all do + clear_config(Pleroma.Formatter) Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) :ok end diff --git a/test/migrations/20200716195806_autolinker_to_linkify_test.exs b/test/migrations/20200716195806_autolinker_to_linkify_test.exs index 063dab0f7..250d11c61 100644 --- a/test/migrations/20200716195806_autolinker_to_linkify_test.exs +++ b/test/migrations/20200716195806_autolinker_to_linkify_test.exs @@ -1,9 +1,10 @@ defmodule Pleroma.Repo.Migrations.AutolinkerToLinkifyTest do use Pleroma.DataCase import Pleroma.Factory - import Pleroma.Tests.Helpers, only: [require_migration: 1] + import Pleroma.Tests.Helpers alias Pleroma.ConfigDB + setup do: clear_config(Pleroma.Formatter) setup_all do: require_migration("20200716195806_autolinker_to_linkify") test "change/0 converts auto_linker opts for Pleroma.Formatter", %{migration: migration} do @@ -13,7 +14,7 @@ test "change/0 converts auto_linker opts for Pleroma.Formatter", %{migration: mi class: false, strip_prefix: false, new_window: false, - rel: "ugc" + rel: "testing" ] insert(:config, group: :auto_linker, key: :opts, value: autolinker_opts) @@ -28,17 +29,20 @@ test "change/0 converts auto_linker opts for Pleroma.Formatter", %{migration: mi class: false, extra: true, new_window: false, - rel: "ugc", + rel: "testing", strip_prefix: false ] + Pleroma.Config.put(Pleroma.Formatter, new_opts) + assert new_opts == Pleroma.Config.get(Pleroma.Formatter) + {text, _mentions, []} = Pleroma.Formatter.linkify( "https://www.businessinsider.com/walmart-will-close-stores-on-thanksgiving-ending-black-friday-tradition-2020-7\n\nOmg will COVID finally end Black Friday???" ) assert text == - "https://www.businessinsider.com/walmart-will-close-stores-on-thanksgiving-ending-black-friday-tradition-2020-7\n\nOmg will COVID finally end Black Friday???" + "https://www.businessinsider.com/walmart-will-close-stores-on-thanksgiving-ending-black-friday-tradition-2020-7\n\nOmg will COVID finally end Black Friday???" end test "transform_opts/1 returns a list of compatible opts", %{migration: migration} do @@ -48,14 +52,14 @@ test "transform_opts/1 returns a list of compatible opts", %{migration: migratio class: false, strip_prefix: false, new_window: false, - rel: "ugc" + rel: "qqq" ] expected_opts = [ class: false, extra: true, new_window: false, - rel: "ugc", + rel: "qqq", strip_prefix: false ] diff --git a/test/migrations/20200722185515_fix_malformed_formatter_config_test.exs b/test/migrations/20200722185515_fix_malformed_formatter_config_test.exs index 9e8f997a0..d3490478e 100644 --- a/test/migrations/20200722185515_fix_malformed_formatter_config_test.exs +++ b/test/migrations/20200722185515_fix_malformed_formatter_config_test.exs @@ -1,9 +1,10 @@ defmodule Pleroma.Repo.Migrations.FixMalformedFormatterConfigTest do use Pleroma.DataCase import Pleroma.Factory - import Pleroma.Tests.Helpers, only: [require_migration: 1] + import Pleroma.Tests.Helpers alias Pleroma.ConfigDB + setup do: clear_config(Pleroma.Formatter) setup_all do: require_migration("20200722185515_fix_malformed_formatter_config") test "change/0 converts a map into a list", %{migration: migration} do @@ -11,7 +12,7 @@ test "change/0 converts a map into a list", %{migration: migration} do class: false, extra: true, new_window: false, - rel: "ugc", + rel: "F", strip_prefix: false } @@ -25,17 +26,20 @@ test "change/0 converts a map into a list", %{migration: migration} do class: false, extra: true, new_window: false, - rel: "ugc", + rel: "F", strip_prefix: false ] + Pleroma.Config.put(Pleroma.Formatter, new_opts) + assert new_opts == Pleroma.Config.get(Pleroma.Formatter) + {text, _mentions, []} = Pleroma.Formatter.linkify( "https://www.businessinsider.com/walmart-will-close-stores-on-thanksgiving-ending-black-friday-tradition-2020-7\n\nOmg will COVID finally end Black Friday???" ) assert text == - "https://www.businessinsider.com/walmart-will-close-stores-on-thanksgiving-ending-black-friday-tradition-2020-7\n\nOmg will COVID finally end Black Friday???" + "https://www.businessinsider.com/walmart-will-close-stores-on-thanksgiving-ending-black-friday-tradition-2020-7\n\nOmg will COVID finally end Black Friday???" end test "change/0 skips if Pleroma.Formatter config is already a list", %{migration: migration} do