From 4e4f77108207157a49a627edb03951e2f15b62f1 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sat, 26 Sep 2020 19:32:16 +0300 Subject: [PATCH] Adjusted MediaProxyControllerTest to gracefully fail on missing dependencies. Installation docs update. Added ffmpeg/imagemagick checks to launch checks (if media preview proxy is enabled). Added documentation on installing optional media / graphics packages (imagemagick, ffmpeg, exiftool). --- docs/installation/alpine_linux_en.md | 18 +++++------ docs/installation/arch_linux_en.md | 13 ++++++-- docs/installation/debian_based_en.md | 13 ++++++-- docs/installation/debian_based_jp.md | 12 +++++-- docs/installation/freebsd_en.md | 8 ++++- docs/installation/gentoo_en.md | 13 ++++++-- docs/installation/netbsd_en.md | 4 +++ docs/installation/openbsd_en.md | 13 ++++++++ docs/installation/openbsd_fi.md | 11 +++++++ .../optional/media_graphics_packages.md | 32 +++++++++++++++++++ docs/installation/otp_en.md | 25 ++++++++++++--- lib/pleroma/application.ex | 18 ++++++++++- lib/pleroma/helpers/media_helper.ex | 12 +++++++ .../media_proxy_controller_test.exs | 15 +++++++++ 14 files changed, 180 insertions(+), 27 deletions(-) create mode 100644 docs/installation/optional/media_graphics_packages.md diff --git a/docs/installation/alpine_linux_en.md b/docs/installation/alpine_linux_en.md index 0b9f5cdae..d89c7f46f 100644 --- a/docs/installation/alpine_linux_en.md +++ b/docs/installation/alpine_linux_en.md @@ -13,8 +13,6 @@ It assumes that you have administrative rights, either as root or a user with [s * `erlang-parsetools` * `erlang-xmerl` * `git` -* `ffmpeg` -* `ImageMagick` * Development Tools * `cmake` @@ -22,6 +20,9 @@ It assumes that you have administrative rights, either as root or a user with [s * `nginx` (preferred, example configs for other reverse proxies can be found in the repo) * `certbot` (or any other ACME client for Let’s Encrypt certificates) +* `ImageMagick` +* `ffmpeg` +* `exiftool` ### Prepare the system @@ -31,7 +32,6 @@ It assumes that you have administrative rights, either as root or a user with [s awk 'NR==2' /etc/apk/repositories | sed 's/main/community/' | tee -a /etc/apk/repositories ``` - * Then update the system, if not already done: ```shell @@ -59,12 +59,6 @@ sudo apk add erlang erlang-runtime-tools erlang-xmerl elixir sudo apk add erlang-eldap ``` -### Install ffmpeg and ImageMagick - -```shell -sudo apk add ffmpeg imagemagick -``` - ### Install PostgreSQL * Install Postgresql server: @@ -85,6 +79,12 @@ sudo /etc/init.d/postgresql start sudo rc-update add postgresql ``` +### Install media / graphics packages (optional, see [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md)) + +```shell +sudo apk add ffmpeg imagemagick exiftool +``` + ### Install PleromaBE * Add a new system user for the Pleroma service: diff --git a/docs/installation/arch_linux_en.md b/docs/installation/arch_linux_en.md index 38bb82432..724b4660a 100644 --- a/docs/installation/arch_linux_en.md +++ b/docs/installation/arch_linux_en.md @@ -10,13 +10,14 @@ This guide will assume that you have administrative rights, either as root or a * `git` * `base-devel` * `cmake` -* `ffmpeg` -* `ImageMagick` #### Optional packages used in this guide * `nginx` (preferred, example configs for other reverse proxies can be found in the repo) * `certbot` (or any other ACME client for Let’s Encrypt certificates) +* `ImageMagick` +* `ffmpeg` +* `exiftool` ### Prepare the system @@ -29,7 +30,7 @@ sudo pacman -Syu * Install some of the above mentioned programs: ```shell -sudo pacman -S git base-devel elixir cmake ffmpeg imagemagick +sudo pacman -S git base-devel elixir cmake ``` ### Install PostgreSQL @@ -54,6 +55,12 @@ sudo -iu postgres initdb -D /var/lib/postgres/data sudo systemctl enable --now postgresql.service ``` +### Install media / graphics packages (optional, see [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md)) + +```shell +sudo pacman -S ffmpeg imagemagick perl-image-exiftool +``` + ### Install PleromaBE * Add a new system user for the Pleroma service: diff --git a/docs/installation/debian_based_en.md b/docs/installation/debian_based_en.md index 30ee64c01..eac499a29 100644 --- a/docs/installation/debian_based_en.md +++ b/docs/installation/debian_based_en.md @@ -13,13 +13,14 @@ This guide will assume you are on Debian Stretch. This guide should also work wi * `git` * `build-essential` * `cmake` -* `ffmpeg` -* `ImageMagick` #### Optional packages used in this guide * `nginx` (preferred, example configs for other reverse proxies can be found in the repo) * `certbot` (or any other ACME client for Let’s Encrypt certificates) +* `ImageMagick` +* `ffmpeg` +* `exiftool` ### Prepare the system @@ -33,7 +34,7 @@ sudo apt full-upgrade * Install some of the above mentioned programs: ```shell -sudo apt install git build-essential postgresql postgresql-contrib cmake ffmpeg imagemagick +sudo apt install git build-essential postgresql postgresql-contrib cmake ``` ### Install Elixir and Erlang @@ -52,6 +53,12 @@ sudo apt update sudo apt install elixir erlang-dev erlang-nox ``` +### Optional packages: [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md) + +```shell +sudo apt install imagemagick ffmpeg libimage-exiftool-perl +``` + ### Install PleromaBE * Add a new system user for the Pleroma service: diff --git a/docs/installation/debian_based_jp.md b/docs/installation/debian_based_jp.md index 219f40525..764afbe1a 100644 --- a/docs/installation/debian_based_jp.md +++ b/docs/installation/debian_based_jp.md @@ -17,13 +17,14 @@ - `git` - `build-essential` - `cmake` -- `ffmpeg` -- `ImageMagick` #### このガイドで利用している追加パッケージ - `nginx` (おすすめです。他のリバースプロキシを使う場合は、参考となる設定をこのリポジトリから探してください) - `certbot` (または何らかのLet's Encrypt向けACMEクライアント) +- `ImageMagick` +- `ffmpeg` +- `exiftool` ### システムを準備する @@ -38,7 +39,6 @@ sudo apt full-upgrade sudo apt install git build-essential postgresql postgresql-contrib cmake ffmpeg imagemagick ``` - ### ElixirとErlangをインストールします * Erlangのリポジトリをダウンロードおよびインストールします。 @@ -53,6 +53,12 @@ sudo apt update sudo apt install elixir erlang-dev erlang-nox ``` +### オプションパッケージ: [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md) + +```shell +sudo apt install imagemagick ffmpeg libimage-exiftool-perl +``` + ### Pleroma BE (バックエンド) をインストールします * Pleroma用に新しいユーザーを作ります。 diff --git a/docs/installation/freebsd_en.md b/docs/installation/freebsd_en.md index 01b410a46..fdcb06c53 100644 --- a/docs/installation/freebsd_en.md +++ b/docs/installation/freebsd_en.md @@ -7,7 +7,7 @@ This document was written for FreeBSD 12.1, but should be work on future release This assumes the target system has `pkg(8)`. ``` -# pkg install elixir postgresql12-server postgresql12-client postgresql12-contrib git-lite sudo nginx gmake acme.sh cmake ffmpeg imagemagick +# pkg install elixir postgresql12-server postgresql12-client postgresql12-contrib git-lite sudo nginx gmake acme.sh cmake ``` Copy the rc.d scripts to the right directory: @@ -26,6 +26,12 @@ Setup the required services to automatically start at boot, using `sysrc(8)`. # service postgresql start ``` +### Install media / graphics packages (optional, see [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md)) + +```shell +# pkg install imagemagick ffmpeg p5-Image-ExifTool +``` + ## Configuring Pleroma Create a user for Pleroma: diff --git a/docs/installation/gentoo_en.md b/docs/installation/gentoo_en.md index 9a8b54ece..638fc4e47 100644 --- a/docs/installation/gentoo_en.md +++ b/docs/installation/gentoo_en.md @@ -29,14 +29,15 @@ Gentoo quite pointedly does not come with a cron daemon installed, and as such i * `dev-lang/elixir` * `dev-vcs/git` * `dev-util/cmake` -* `media-video/ffmpeg` -* `media-gfx/imagemagick` #### Optional ebuilds used in this guide * `www-servers/nginx` (preferred, example configs for other reverse proxies can be found in the repo) * `app-crypt/certbot` (or any other ACME client for Let’s Encrypt certificates) * `app-crypt/certbot-nginx` (nginx certbot plugin that allows use of the all-powerful `--nginx` flag on certbot) +* `media-gfx/imagemagick` +* `media-video/ffmpeg` +* `media-libs/exiftool` ### Prepare the system @@ -49,7 +50,7 @@ Gentoo quite pointedly does not come with a cron daemon installed, and as such i * Emerge all required the required and suggested software in one go: ```shell - # emerge --ask dev-db/postgresql dev-lang/elixir dev-vcs/git www-servers/nginx app-crypt/certbot app-crypt/certbot-nginx dev-util/cmake media-video/ffmpeg media-gfx/imagemagick + # emerge --ask dev-db/postgresql dev-lang/elixir dev-vcs/git www-servers/nginx app-crypt/certbot app-crypt/certbot-nginx dev-util/cmake ``` If you would not like to install the optional packages, remove them from this line. @@ -89,6 +90,12 @@ If you do not plan to make any modifications to your Pleroma instance, cloning d Not only does this make it much easier to deploy changes you make, as you can commit and pull from upstream and all that good stuff from the comfort of your local machine then simply `git pull` on your instance server when you're ready to deploy, it also ensures you are compliant with the Affero General Public Licence that Pleroma is licenced under, which stipulates that all network services provided with modified AGPL code must publish their changes on a publicly available internet service and for free. It also makes it much easier to ask for help from and provide help to your fellow Pleroma admins if your public repo always reflects what you are running because it is part of your deployment procedure. +### Install media / graphics packages (optional, see [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md)) + +```shell +# emerge --ask media-video/ffmpeg media-gfx/imagemagick media-libs/exiftool +``` + ### Install PleromaBE * Add a new system user for the Pleroma service and set up default directories: diff --git a/docs/installation/netbsd_en.md b/docs/installation/netbsd_en.md index 8cf2e4682..d5fa04fdf 100644 --- a/docs/installation/netbsd_en.md +++ b/docs/installation/netbsd_en.md @@ -44,6 +44,10 @@ pgsql=YES First, run `# /etc/rc.d/pgsql start`. Then, `$ sudo -Hu pgsql -g pgsql createdb`. +### Install media / graphics packages (optional, see [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md)) + +`# pkgin install ImageMagick ffmpeg4 p5-Image-ExifTool` + ## Configuring Pleroma Create a user for Pleroma: diff --git a/docs/installation/openbsd_en.md b/docs/installation/openbsd_en.md index 277e5ec43..8092ac379 100644 --- a/docs/installation/openbsd_en.md +++ b/docs/installation/openbsd_en.md @@ -25,6 +25,19 @@ pkg_add elixir gmake git postgresql-server postgresql-contrib cmake ffmpeg Image Pleroma requires a reverse proxy, OpenBSD has relayd in base (and is used in this guide) and packages/ports are available for nginx (www/nginx) and apache (www/apache-httpd). Independently of the reverse proxy, [acme-client(1)](https://man.openbsd.org/acme-client) can be used to get a certificate from Let's Encrypt. +#### Optional software + +Per [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md): + * ImageMagick + * ffmpeg + * exiftool + +To install the above: + +``` +pkg_add ImageMagick ffmpeg p5-Image-ExifTool +``` + #### Creating the pleroma user Pleroma will be run by a dedicated user, \_pleroma. Before creating it, insert the following lines in login.conf: ``` diff --git a/docs/installation/openbsd_fi.md b/docs/installation/openbsd_fi.md index 2cb03e73e..01cf34ab4 100644 --- a/docs/installation/openbsd_fi.md +++ b/docs/installation/openbsd_fi.md @@ -18,6 +18,17 @@ Asenna tarvittava ohjelmisto: `# pkg_add git elixir gmake postgresql-server-10.3 postgresql-contrib-10.3 cmake ffmpeg ImageMagick` +#### Optional software + +[`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md): + * ImageMagick + * ffmpeg + * exiftool + +Asenna tarvittava ohjelmisto: + +`# pkg_add ImageMagick ffmpeg p5-Image-ExifTool` + Luo postgresql-tietokanta: `# su - _postgresql` diff --git a/docs/installation/optional/media_graphics_packages.md b/docs/installation/optional/media_graphics_packages.md new file mode 100644 index 000000000..cb3d71188 --- /dev/null +++ b/docs/installation/optional/media_graphics_packages.md @@ -0,0 +1,32 @@ +# Optional software packages needed for specific functionality + +For specific Pleroma functionality (which is disabled by default) some or all of the below packages are required: + * `ImageMagic` + * `ffmpeg` + * `exiftool` + +Please refer to documentation in `docs/installation` on how to install them on specific OS. + +Note: the packages are not required with the current default settings of Pleroma. + +## `ImageMagick` + +`ImageMagick` is a set of tools to create, edit, compose, or convert bitmap images. + +It is required for the following Pleroma features: + * `Pleroma.Upload.Filters.Mogrify`, `Pleroma.Upload.Filters.Mogrifun` upload filters (related config: `Plaroma.Upload/filters` in `config/config.exs`) + * Media preview proxy for still images (related config: `media_preview_proxy/enabled` in `config/config.exs`) + +## `ffmpeg` + +`ffmpeg` is software to record, convert and stream audio and video. + +It is required for the following Pleroma features: + * Media preview proxy for videos (related config: `media_preview_proxy/enabled` in `config/config.exs`) + +## `exiftool` + +`exiftool` is media files metadata reader/writer. + +It is required for the following Pleroma features: + * `Pleroma.Upload.Filters.Exiftool` upload filter (related config: `Plaroma.Upload/filters` in `config/config.exs`) diff --git a/docs/installation/otp_en.md b/docs/installation/otp_en.md index 8fdf6b5b5..f6b05c887 100644 --- a/docs/installation/otp_en.md +++ b/docs/installation/otp_en.md @@ -27,19 +27,36 @@ Other than things bundled in the OTP release Pleroma depends on: * PostgreSQL (also utilizes extensions in postgresql-contrib) * nginx (could be swapped with another reverse proxy but this guide covers only it) * certbot (for Let's Encrypt certificates, could be swapped with another ACME client, but this guide covers only it) -* ffmpeg (needed for media preview proxy) -* ImageMagick (needed for media preview proxy) === "Alpine" ``` echo "http://nl.alpinelinux.org/alpine/latest-stable/community" >> /etc/apk/repositories apk update - apk add curl unzip ncurses postgresql postgresql-contrib nginx certbot ffmpeg imagemagick + apk add curl unzip ncurses postgresql postgresql-contrib nginx certbot ``` === "Debian/Ubuntu" ``` - apt install curl unzip libncurses5 postgresql postgresql-contrib nginx certbot ffmpeg imagemagick + apt install curl unzip libncurses5 postgresql postgresql-contrib nginx certbot + ``` + +### Installing optional packages + +Per [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md): + * ImageMagick + * ffmpeg + * exiftool + +=== "Alpine" + ``` + echo "http://nl.alpinelinux.org/alpine/latest-stable/community" >> /etc/apk/repositories + apk update + apk add imagemagick ffmpeg exiftool + ``` + +=== "Debian/Ubuntu" + ``` + apt install imagemagick ffmpeg libimage-exiftool-perl ``` ## Setup diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 00ec79a2a..d7d8e423e 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -268,7 +268,8 @@ defp check_system_commands do with true <- filter in filters, false <- Pleroma.Utils.command_available?(command_required) do Logger.error( - "#{filter} is specified in list of Pleroma.Upload filters, but the #{command_required} command is not found" + "#{filter} is specified in list of Pleroma.Upload filters, but the " <> + "#{command_required} command is not found" ) end end @@ -276,5 +277,20 @@ defp check_system_commands do check_filter.(Pleroma.Upload.Filters.Exiftool, "exiftool") check_filter.(Pleroma.Upload.Filters.Mogrify, "mogrify") check_filter.(Pleroma.Upload.Filters.Mogrifun, "mogrify") + + with true <- Config.get([:media_preview_proxy, :enabled]), + missing_graphics_tools = Pleroma.Helpers.MediaHelper.missing_dependencies(), + [] <- missing_graphics_tools do + :noop + else + false -> + :noop + + missing_graphics_tools -> + Logger.error( + "The following dependencies required by Media preview proxy " <> + "(which is currently enabled) are not installed: #{inspect(missing_graphics_tools)}" + ) + end end end diff --git a/lib/pleroma/helpers/media_helper.ex b/lib/pleroma/helpers/media_helper.ex index b6f35a24b..6b799173e 100644 --- a/lib/pleroma/helpers/media_helper.ex +++ b/lib/pleroma/helpers/media_helper.ex @@ -9,6 +9,18 @@ defmodule Pleroma.Helpers.MediaHelper do alias Pleroma.HTTP + require Logger + + def missing_dependencies do + Enum.reduce([imagemagick: "convert", ffmpeg: "ffmpeg"], [], fn {sym, executable}, acc -> + if Pleroma.Utils.command_available?(executable) do + acc + else + [sym | acc] + end + end) + end + def image_resize(url, options) do with executable when is_binary(executable) <- System.find_executable("convert"), {:ok, args} <- prepare_image_resize_args(options), diff --git a/test/web/media_proxy/media_proxy_controller_test.exs b/test/web/media_proxy/media_proxy_controller_test.exs index 33e6873f7..e9b584822 100644 --- a/test/web/media_proxy/media_proxy_controller_test.exs +++ b/test/web/media_proxy/media_proxy_controller_test.exs @@ -81,6 +81,15 @@ test "it returns 404 when url is in banned_urls cache", %{conn: conn, url: url} end describe "Media Preview Proxy" do + def assert_dependencies_installed do + missing_dependencies = Pleroma.Helpers.MediaHelper.missing_dependencies() + + assert missing_dependencies == [], + "Error: missing dependencies (please refer to `docs/installation`): #{ + inspect(missing_dependencies) + }" + end + setup do clear_config([:media_proxy, :enabled], true) clear_config([:media_preview_proxy, :enabled], true) @@ -184,6 +193,8 @@ test "with `static=true` and GIF image preview requested, responds with JPEG ima url: url, media_proxy_url: media_proxy_url } do + assert_dependencies_installed() + # Setting a high :min_content_length to ensure this scenario is not affected by its logic clear_config([:media_preview_proxy, :min_content_length], 1_000_000_000) @@ -270,6 +281,8 @@ test "thumbnails PNG images into PNG", %{ url: url, media_proxy_url: media_proxy_url } do + assert_dependencies_installed() + Tesla.Mock.mock(fn %{method: "head", url: ^media_proxy_url} -> %Tesla.Env{status: 200, body: "", headers: [{"content-type", "image/png"}]} @@ -290,6 +303,8 @@ test "thumbnails JPEG images into JPEG", %{ url: url, media_proxy_url: media_proxy_url } do + assert_dependencies_installed() + Tesla.Mock.mock(fn %{method: "head", url: ^media_proxy_url} -> %Tesla.Env{status: 200, body: "", headers: [{"content-type", "image/jpeg"}]}