Uploads fun, part. 2

This commit is contained in:
href 2018-11-29 21:11:45 +01:00
parent 97b00d366f
commit 02d3dc6869
No known key found for this signature in database
GPG Key ID: EE8296C1A152C325
17 changed files with 491 additions and 265 deletions

View File

@ -10,11 +10,19 @@
config :pleroma, Pleroma.Repo, types: Pleroma.PostgresTypes config :pleroma, Pleroma.Repo, types: Pleroma.PostgresTypes
# Upload configuration
config :pleroma, Pleroma.Upload, config :pleroma, Pleroma.Upload,
uploader: Pleroma.Uploaders.Local, uploader: Pleroma.Uploaders.Local,
strip_exif: false, # filters: [Pleroma.Upload.DedupeFilter, Pleroma.Upload.MogrifyFilter],
filters: [],
proxy_remote: false, proxy_remote: false,
proxy_opts: [inline_content_types: true, keep_user_agent: true] proxy_opts: []
# Strip Exif
# Also put Pleroma.Upload.MogrifyFilter in the `filters` list of Pleroma.Upload configuration.
# config :pleroma, Pleroma.Upload.MogrifyFilter,
# args: "strip"
# Pleroma.Upload.MogrifyFilter: [args: "strip"]
config :pleroma, Pleroma.Uploaders.Local, uploads: "uploads" config :pleroma, Pleroma.Uploaders.Local, uploads: "uploads"

View File

@ -34,33 +34,50 @@ def run([target_uploader | args]) do
:timer.sleep(:timer.seconds(5)) :timer.sleep(:timer.seconds(5))
end end
uploads = File.ls!(local_path) uploads =
File.ls!(local_path)
|> Enum.map(fn id ->
root_path = Path.join(local_path, id)
cond do
File.dir?(root_path) ->
files = for file <- File.ls!(root_path), do: {id, file, Path.join([root_path, file])}
case List.first(files) do
{id, file, path} ->
{%Pleroma.Upload{id: id, name: file, path: id <> "/" <> file, tempfile: path},
root_path}
_ ->
nil
end
File.exists?(root_path) ->
file = Path.basename(id)
[hash, ext] = String.split(id, ".")
{%Pleroma.Upload{id: hash, name: file, path: file, tempfile: root_path}, root_path}
true ->
nil
end
end)
|> Enum.filter(& &1)
total_count = length(uploads) total_count = length(uploads)
Logger.info("Found #{total_count} uploads")
uploads uploads
|> Task.async_stream( |> Task.async_stream(
fn uuid -> fn {upload, root_path} ->
u_path = Path.join(local_path, uuid) case Upload.store(upload, uploader: uploader, filters: [], size_limit: nil) do
{:ok, _} ->
if delete?, do: File.rm_rf!(root_path)
Logger.debug("uploaded: #{inspect(upload.path)} #{inspect(upload)}")
:ok
{name, path} = error ->
cond do Logger.error("failed to upload #{inspect(upload.path)}: #{inspect(error)}")
File.dir?(u_path) ->
files = for file <- File.ls!(u_path), do: {{file, uuid}, Path.join([u_path, file])}
List.first(files)
File.exists?(u_path) ->
# {uuid, u_path}
raise "should_dedupe local storage not supported yet sorry"
end
{:ok, _} =
Upload.store({:from_local, name, path}, should_dedupe: false, uploader: uploader)
if delete? do
File.rm_rf!(u_path)
end end
Logger.debug("uploaded: #{inspect(name)}")
end, end,
timeout: 150_000 timeout: 150_000
) )
@ -75,6 +92,6 @@ def run([target_uploader | args]) do
end end
def run(_) do def run(_) do
Logger.error("Usage: migrate_local_uploads UploaderName [--delete]") Logger.error("Usage: migrate_local_uploads S3|Swift [--delete]")
end end
end end

100
lib/pleroma/mime.ex Normal file
View File

@ -0,0 +1,100 @@
defmodule Pleroma.MIME do
@moduledoc """
Returns the mime-type of a binary and optionally a normalized file-name. Requires at least (the first) 8 bytes.
"""
@default "application/octet-stream"
@spec file_mime_type(String.t()) ::
{:ok, content_type :: String.t(), filename :: String.t()} | {:error, any()} | :error
def file_mime_type(path, filename) do
with {:ok, content_type} <- file_mime_type(path),
filename <- fix_extension(filename, content_type) do
{:ok, content_type, filename}
end
end
@spec file_mime_type(String.t()) :: {:ok, String.t()} | {:error, any()} | :error
def file_mime_type(filename) do
File.open(filename, [:read], fn f ->
check_mime_type(IO.binread(f, 8))
end)
end
def bin_mime_type(binary, filename) do
with {:ok, content_type} <- bin_mime_type(binary),
filename <- fix_extension(filename, content_type) do
{:ok, content_type, filename}
end
end
@spec bin_mime_type(binary()) :: {:ok, String.t()} | :error
def bin_mime_type(<<head::binary-size(8), _::binary>>) do
{:ok, check_mime_type(head)}
end
def mime_type(<<_::binary>>), do: {:ok, @default}
def bin_mime_type(_), do: :error
defp fix_extension(filename, content_type) do
parts = String.split(filename, ".")
new_filename =
if length(parts) > 1 do
Enum.drop(parts, -1) |> Enum.join(".")
else
Enum.join(parts)
end
cond do
content_type == "application/octet-stream" ->
filename
ext = List.first(MIME.extensions(content_type)) ->
new_filename <> "." <> ext
true ->
Enum.join([new_filename, String.split(content_type, "/") |> List.last()], ".")
end
end
defp check_mime_type(<<0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A>>) do
"image/png"
end
defp check_mime_type(<<0x47, 0x49, 0x46, 0x38, _, 0x61, _, _>>) do
"image/gif"
end
defp check_mime_type(<<0xFF, 0xD8, 0xFF, _, _, _, _, _>>) do
"image/jpeg"
end
defp check_mime_type(<<0x1A, 0x45, 0xDF, 0xA3, _, _, _, _>>) do
"video/webm"
end
defp check_mime_type(<<0x00, 0x00, 0x00, _, 0x66, 0x74, 0x79, 0x70>>) do
"video/mp4"
end
defp check_mime_type(<<0x49, 0x44, 0x33, _, _, _, _, _>>) do
"audio/mpeg"
end
defp check_mime_type(<<255, 251, _, 68, 0, 0, 0, 0>>) do
"audio/mpeg"
end
defp check_mime_type(<<0x4F, 0x67, 0x67, 0x53, 0x00, 0x02, 0x00, 0x00>>) do
"audio/ogg"
end
defp check_mime_type(<<0x52, 0x49, 0x46, 0x46, _, _, _, _>>) do
"audio/wav"
end
defp check_mime_type(_) do
@default
end
end

View File

@ -101,7 +101,7 @@ def call(conn = %{method: method}, url, opts \\ []) when method in @methods do
end end
with {:ok, code, headers, client} <- request(method, url, req_headers, hackney_opts), with {:ok, code, headers, client} <- request(method, url, req_headers, hackney_opts),
:ok <- header_lenght_constraint(headers, Keyword.get(opts, :max_body_length)) do :ok <- header_length_constraint(headers, Keyword.get(opts, :max_body_length)) do
response(conn, client, url, code, headers, opts) response(conn, client, url, code, headers, opts)
else else
{:ok, code, headers} -> {:ok, code, headers} ->
@ -298,7 +298,7 @@ defp build_resp_content_disposition_header(headers, opts) do
end end
end end
defp header_lenght_constraint(headers, limit) when is_integer(limit) and limit > 0 do defp header_length_constraint(headers, limit) when is_integer(limit) and limit > 0 do
with {_, size} <- List.keyfind(headers, "content-length", 0), with {_, size} <- List.keyfind(headers, "content-length", 0),
{size, _} <- Integer.parse(size), {size, _} <- Integer.parse(size),
true <- size <= limit do true <- size <= limit do
@ -312,7 +312,7 @@ defp header_lenght_constraint(headers, limit) when is_integer(limit) and limit >
end end
end end
defp header_lenght_constraint(_, _), do: :ok defp header_length_constraint(_, _), do: :ok
defp body_size_constraint(size, limit) when is_integer(limit) and limit > 0 and size >= limit do defp body_size_constraint(size, limit) when is_integer(limit) and limit > 0 and size >= limit do
{:error, :body_too_large} {:error, :body_too_large}

View File

@ -1,31 +1,73 @@
defmodule Pleroma.Upload do defmodule Pleroma.Upload do
@moduledoc """
# Upload
Options:
* `:type`: presets for activity type (defaults to Document) and size limits from app configuration
* `:description`: upload alternative text
* `:uploader`: override uploader
* `:filters`: override filters
* `:size_limit`: override size limit
* `:activity_type`: override activity type
The `%Pleroma.Upload{}` struct: all documented fields are meant to be overwritten in filters:
* `:id` - the upload id.
* `:name` - the upload file name.
* `:path` - the upload path: set at first to `id/name` but can be changed. Keep in mind that the path
is once created permanent and changing it (especially in uploaders) is probably a bad idea!
* `:tempfile` - path to the temporary file. Prefer in-place changes on the file rather than changing the
path as the temporary file is also tracked by `Plug.Upload{}` and automatically deleted once the request is over.
Related behaviors:
* `Pleroma.Uploaders.Uploader`
* `Pleroma.Upload.Filter`
"""
alias Ecto.UUID alias Ecto.UUID
require Logger require Logger
@type upload_option :: @type source ::
{:dedupe, boolean()} | {:size_limit, non_neg_integer()} | {:uploader, module()} Plug.Upload.t() | data_uri_string ::
@type upload_source :: String.t() | {:from_local, name :: String.t(), id :: String.t(), path :: String.t()}
Plug.Upload.t() | data_uri_string() ::
String.t() | {:from_local, name :: String.t(), uuid :: String.t(), path :: String.t()}
@spec store(upload_source, options :: [upload_option()]) :: {:ok, Map.t()} | {:error, any()} @type option ::
{:type, :avatar | :banner | :background}
| {:description, String.t()}
| {:activity_type, String.t()}
| {:size_limit, nil | non_neg_integer()}
| {:uploader, module()}
| {:filters, [module()]}
@type t :: %__MODULE__{
id: String.t(),
name: String.t(),
tempfile: String.t(),
content_type: String.t(),
path: String.t()
}
defstruct [:id, :name, :tempfile, :content_type, :path]
@spec store(source, options :: [option()]) :: {:ok, Map.t()} | {:error, any()}
def store(upload, opts \\ []) do def store(upload, opts \\ []) do
opts = get_opts(opts) opts = get_opts(opts)
with {:ok, name, uuid, path, content_type} <- process_upload(upload, opts), with {:ok, upload} <- prepare_upload(upload, opts),
_ <- strip_exif_data(content_type, path), upload = %__MODULE__{upload | path: upload.path || "#{upload.id}/#{upload.name}"},
{:ok, url_spec} <- opts.uploader.put_file(name, uuid, path, content_type, opts) do {:ok, upload} <- Pleroma.Upload.Filter.filter(opts.filters, upload),
{:ok, url_spec} <- Pleroma.Uploaders.Uploader.put_file(opts.uploader, upload) do
{:ok, {:ok,
%{ %{
"type" => "Image", "type" => opts.activity_type,
"url" => [ "url" => [
%{ %{
"type" => "Link", "type" => "Link",
"mediaType" => content_type, "mediaType" => upload.content_type,
"href" => url_from_spec(url_spec) "href" => url_from_spec(url_spec)
} }
], ],
"name" => name "name" => Map.get(opts, :description) || upload.name
}} }}
else else
{:error, error} -> {:error, error} ->
@ -38,40 +80,98 @@ def store(upload, opts \\ []) do
end end
defp get_opts(opts) do defp get_opts(opts) do
%{ {size_limit, activity_type} =
dedupe: Keyword.get(opts, :dedupe, Pleroma.Config.get([:instance, :dedupe_media])), case Keyword.get(opts, :type) do
size_limit: Keyword.get(opts, :size_limit, Pleroma.Config.get([:instance, :upload_limit])), :banner ->
uploader: Keyword.get(opts, :uploader, Pleroma.Config.get([__MODULE__, :uploader])) {Pleroma.Config.get!([:instance, :banner_upload_limit]), "Image"}
:avatar ->
{Pleroma.Config.get!([:instance, :avatar_upload_limit]), "Image"}
:background ->
{Pleroma.Config.get!([:instance, :background_upload_limit]), "Image"}
_ ->
{Pleroma.Config.get!([:instance, :upload_limit]), "Document"}
end
opts = %{
activity_type: Keyword.get(opts, :activity_type, activity_type),
size_limit: Keyword.get(opts, :size_limit, size_limit),
uploader: Keyword.get(opts, :uploader, Pleroma.Config.get([__MODULE__, :uploader])),
filters: Keyword.get(opts, :filters, Pleroma.Config.get([__MODULE__, :filters])),
description: Keyword.get(opts, :description)
} }
# TODO: 1.0+ : remove old config compatibility
opts =
if Pleroma.Config.get([__MODULE__, :strip_exif]) == true &&
!Enum.member?(opts.filters, Pleroma.Upload.Filter.Mogrify) do
Logger.warn("""
Pleroma: configuration `:instance, :strip_exif` is deprecated, please instead set:
:instance, Pleroma.Upload, [filters: [Pleroma.Upload.Filter.Mogrify]]
:pleroma, Pleroma.Upload.Mogrify, args: "strip"
""")
Pleroma.Config.put([Pleroma.Upload.Filter.Mogrify], args: "strip")
Map.put(opts, :filters, opts.filters ++ [Pleroma.Upload.Filter.Mogrify])
else
opts
end
opts =
if Pleroma.Config.get([:instance, :dedupe_media]) == true &&
!Enum.member?(opts.filters, Pleroma.Upload.Filter.Dedupe) do
Logger.warn("""
Pleroma: configuration `:instance, :dedupe_media` is deprecated, please instead set:
:instance, Pleroma.Upload, [filters: [Pleroma.Upload.Filter.Dedupe]]
""")
Map.put(opts, :filters, opts.filters ++ [Pleroma.Upload.Filter.Dedupe])
else
opts
end
end end
defp process_upload(%Plug.Upload{} = file, opts) do defp prepare_upload(%Plug.Upload{} = file, opts) do
with :ok <- check_file_size(file.path, opts.size_limit), with :ok <- check_file_size(file.path, opts.size_limit),
uuid <- get_uuid(file, opts.dedupe), {:ok, content_type, name} <- Pleroma.MIME.file_mime_type(file.path, file.filename) do
content_type <- get_content_type(file.path), {:ok,
name <- get_name(file, uuid, content_type, opts.dedupe) do %__MODULE__{
{:ok, name, uuid, file.path, content_type} id: UUID.generate(),
name: name,
tempfile: file.path,
content_type: content_type
}}
end end
end end
defp process_upload(%{"img" => "data:image/" <> image_data}, opts) do defp prepare_upload(%{"img" => "data:image/" <> image_data}, opts) do
parsed = Regex.named_captures(~r/(?<filetype>jpeg|png|gif);base64,(?<data>.*)/, image_data) parsed = Regex.named_captures(~r/(?<filetype>jpeg|png|gif);base64,(?<data>.*)/, image_data)
data = Base.decode64!(parsed["data"], ignore: :whitespace) data = Base.decode64!(parsed["data"], ignore: :whitespace)
hash = String.downcase(Base.encode16(:crypto.hash(:sha256, data))) hash = String.downcase(Base.encode16(:crypto.hash(:sha256, data)))
with :ok <- check_binary_size(data, opts.size_limit), with :ok <- check_binary_size(data, opts.size_limit),
tmp_path <- tempfile_for_image(data), tmp_path <- tempfile_for_image(data),
content_type <- get_content_type(tmp_path), {:ok, content_type, name} <-
uuid <- UUID.generate(), Pleroma.MIME.bin_mime_type(data, hash <> "." <> parsed["filetype"]) do
name <- create_name(hash, parsed["filetype"], content_type) do {:ok,
{:ok, name, uuid, tmp_path, content_type} %__MODULE__{
id: UUID.generate(),
name: name,
tempfile: tmp_path,
content_type: content_type
}}
end end
end end
# For Mix.Tasks.MigrateLocalUploads # For Mix.Tasks.MigrateLocalUploads
defp process_upload({:from_local, name, uuid, path}, _opts) do defp prepare_upload(upload = %__MODULE__{tempfile: path}, _opts) do
with content_type <- get_content_type(path) do with {:ok, content_type} <- Pleroma.MIME.file_mime_type(path) do
{:ok, name, uuid, path, content_type} {:ok, %__MODULE__{upload | content_type: content_type}}
end end
end end
@ -104,119 +204,6 @@ defp tempfile_for_image(data) do
tmp_path tmp_path
end end
defp strip_exif_data(content_type, file) do
settings = Application.get_env(:pleroma, Pleroma.Upload)
do_strip = Keyword.fetch!(settings, :strip_exif)
[filetype, _ext] = String.split(content_type, "/")
if filetype == "image" and do_strip == true do
Mogrify.open(file) |> Mogrify.custom("strip") |> Mogrify.save(in_place: true)
end
end
defp create_name(uuid, ext, type) do
extension =
cond do
type == "application/octect-stream" -> ext
ext = mime_extension(ext) -> ext
true -> String.split(type, "/") |> List.last()
end
[uuid, extension]
|> Enum.join(".")
|> String.downcase()
end
defp mime_extension(type) do
List.first(MIME.extensions(type))
end
defp get_uuid(file, should_dedupe) do
if should_dedupe do
Base.encode16(:crypto.hash(:sha256, File.read!(file.path)))
else
UUID.generate()
end
end
defp get_name(file, uuid, type, should_dedupe) do
if should_dedupe do
create_name(uuid, List.last(String.split(file.filename, ".")), type)
else
parts = String.split(file.filename, ".")
new_filename =
if length(parts) > 1 do
Enum.drop(parts, -1) |> Enum.join(".")
else
Enum.join(parts)
end
cond do
type == "application/octet-stream" ->
file.filename
ext = mime_extension(type) ->
new_filename <> "." <> ext
true ->
Enum.join([new_filename, String.split(type, "/") |> List.last()], ".")
end
end
end
def get_content_type(file) do
match =
File.open(file, [:read], fn f ->
case IO.binread(f, 8) do
<<0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A>> ->
"image/png"
<<0x47, 0x49, 0x46, 0x38, _, 0x61, _, _>> ->
"image/gif"
<<0xFF, 0xD8, 0xFF, _, _, _, _, _>> ->
"image/jpeg"
<<0x1A, 0x45, 0xDF, 0xA3, _, _, _, _>> ->
"video/webm"
<<0x00, 0x00, 0x00, _, 0x66, 0x74, 0x79, 0x70>> ->
"video/mp4"
<<0x49, 0x44, 0x33, _, _, _, _, _>> ->
"audio/mpeg"
<<255, 251, _, 68, 0, 0, 0, 0>> ->
"audio/mpeg"
<<0x4F, 0x67, 0x67, 0x53, 0x00, 0x02, 0x00, 0x00>> ->
case IO.binread(f, 27) do
<<_::size(160), 0x80, 0x74, 0x68, 0x65, 0x6F, 0x72, 0x61>> ->
"video/ogg"
_ ->
"audio/ogg"
end
<<0x52, 0x49, 0x46, 0x46, _, _, _, _>> ->
"audio/wav"
_ ->
"application/octet-stream"
end
end)
case match do
{:ok, type} -> type
_e -> "application/octet-stream"
end
end
defp uploader() do
Pleroma.Config.get!([Pleroma.Upload, :uploader])
end
defp url_from_spec({:file, path}) do defp url_from_spec({:file, path}) do
[Pleroma.Web.base_url(), "media", path] [Pleroma.Web.base_url(), "media", path]
|> Path.join() |> Path.join()

View File

@ -0,0 +1,35 @@
defmodule Pleroma.Upload.Filter do
@moduledoc """
Upload Filter behaviour
This behaviour allows to run filtering actions just before a file is uploaded. This allows to:
* morph in place the temporary file
* change any field of a `Pleroma.Upload` struct
* cancel/stop the upload
"""
require Logger
@callback filter(Pleroma.Upload.t()) :: :ok | {:ok, Pleroma.Upload.t()} | {:error, any()}
@spec filter([module()], Pleroma.Upload.t()) :: {:ok, Pleroma.Upload.t()} | {:error, any()}
def filter([], upload) do
{:ok, upload}
end
def filter([filter | rest], upload) do
case filter.filter(upload) do
:ok ->
filter(rest, upload)
{:ok, upload} ->
filter(rest, upload)
error ->
Logger.error("#{__MODULE__}: Filter #{filter} failed: #{inspect(error)}")
error
end
end
end

View File

@ -0,0 +1,10 @@
defmodule Pleroma.Upload.Filter.Dedupe do
@behaviour Pleroma.Upload.Filter
def filter(upload = %Pleroma.Upload{name: name, tempfile: path}) do
extension = String.split(name, ".") |> List.last()
shasum = :crypto.hash(:sha256, File.read!(upload.tempfile)) |> Base.encode16(case: :lower)
filename = shasum <> "." <> extension
{:ok, %Pleroma.Upload{upload | id: shasum, path: filename}}
end
end

View File

@ -0,0 +1,60 @@
defmodule Pleroma.Upload.Filter.Mogrifun do
@behaviour Pleroma.Upload.Filter
@filters [
{"implode", "1"},
{"-raise", "20"},
{"+raise", "20"},
[{"-interpolate", "nearest"}, {"-virtual-pixel", "mirror"}, {"-spread", "5"}],
"+polaroid",
{"-statistic", "Mode 10"},
{"-emboss", "0x1.1"},
{"-emboss", "0x2"},
{"-colorspace", "Gray"},
"-negate",
[{"-channel", "green"}, "-negate"],
[{"-channel", "red"}, "-negate"],
[{"-channel", "blue"}, "-negate"],
{"+level-colors", "green,gold"},
{"+level-colors", ",DodgerBlue"},
{"+level-colors", ",Gold"},
{"+level-colors", ",Lime"},
{"+level-colors", ",Red"},
{"+level-colors", ",DarkGreen"},
{"+level-colors", "firebrick,yellow"},
{"+level-colors", "'rgb(102,75,25)',lemonchiffon"},
[{"fill", "red"}, {"tint", "40"}],
[{"fill", "green"}, {"tint", "40"}],
[{"fill", "blue"}, {"tint", "40"}],
[{"fill", "yellow"}, {"tint", "40"}]
]
def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do
filter = Enum.random(@filters)
file
|> Mogrify.open()
|> mogrify_filter(filter)
|> Mogrify.save(in_place: true)
:ok
end
def filter(_), do: :ok
defp mogrify_filter(mogrify, [filter | rest]) do
mogrify
|> mogrify_filter(filter)
|> mogrify_filter(rest)
end
defp mogrify_filter(mogrify, []), do: mogrify
defp mogrify_filter(mogrify, {action, options}) do
Mogrify.custom(mogrify, action, options)
end
defp mogrify_filter(mogrify, string) when is_binary(string) do
Mogrify.custom(mogrify, string)
end
end

View File

@ -0,0 +1,37 @@
defmodule Pleroma.Upload.Filter.Mogrify do
@behaviour Pleroma.Uploader.Filter
@type conversion :: action :: String.t() | {action :: String.t(), opts :: String.t()}
@type conversions :: conversion() | [conversion()]
def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do
filters = Pleroma.Config.get!([__MODULE__, :args])
file
|> Mogrify.open()
|> mogrify_filter(filters)
|> Mogrify.save(in_place: true)
:ok
end
def filter(_), do: :ok
defp mogrify_filter(mogrify, nil), do: mogrify
defp mogrify_filter(mogrify, [filter | rest]) do
mogrify
|> mogrify_filter(filter)
|> mogrify_filter(rest)
end
defp mogrify_filter(mogrify, []), do: mogrify
defp mogrify_filter(mogrify, {action, options}) do
Mogrify.custom(mogrify, action, options)
end
defp mogrify_filter(mogrify, action) when is_binary(action) do
Mogrify.custom(mogrify, action)
end
end

View File

@ -7,39 +7,28 @@ def get_file(_) do
{:ok, {:static_dir, upload_path()}} {:ok, {:static_dir, upload_path()}}
end end
def put_file(name, uuid, tmpfile, _content_type, opts) do def put_file(upload) do
upload_folder = get_upload_path(uuid, opts.dedupe) {local_path, file} =
case Enum.reverse(String.split(upload.path, "/", trim: true)) do
[file] ->
{upload_path(), file}
File.mkdir_p!(upload_folder) [file | folders] ->
path = Path.join([upload_path()] ++ Enum.reverse(folders))
File.mkdir_p!(path)
{path, file}
end
result_file = Path.join(upload_folder, name) result_file = Path.join(local_path, file)
if File.exists?(result_file) do unless File.exists?(result_file) do
File.rm!(tmpfile) File.cp!(upload.tempfile, result_file)
else
File.cp!(tmpfile, result_file)
end end
{:ok, {:file, get_url(name, uuid, opts.dedupe)}} :ok
end end
def upload_path do def upload_path do
Pleroma.Config.get!([__MODULE__, :uploads]) Pleroma.Config.get!([__MODULE__, :uploads])
end end
defp get_upload_path(uuid, should_dedupe) do
if should_dedupe do
upload_path()
else
Path.join(upload_path(), uuid)
end
end
defp get_url(name, uuid, should_dedupe) do
if should_dedupe do
:cow_uri.urlencode(name)
else
Path.join(uuid, :cow_uri.urlencode(name))
end
end
end end

View File

@ -11,22 +11,21 @@ def get_file(file) do
Pleroma.Uploaders.Local.get_file(file) Pleroma.Uploaders.Local.get_file(file)
end end
def put_file(name, uuid, path, content_type, opts) do def put_file(upload) do
cgi = Pleroma.Config.get([Pleroma.Uploaders.MDII, :cgi]) cgi = Pleroma.Config.get([Pleroma.Uploaders.MDII, :cgi])
files = Pleroma.Config.get([Pleroma.Uploaders.MDII, :files]) files = Pleroma.Config.get([Pleroma.Uploaders.MDII, :files])
{:ok, file_data} = File.read(path) {:ok, file_data} = File.read(upload.tempfile)
extension = String.split(name, ".") |> List.last() extension = String.split(upload.name, ".") |> List.last()
query = "#{cgi}?#{extension}" query = "#{cgi}?#{extension}"
with {:ok, %{status_code: 200, body: body}} <- @httpoison.post(query, file_data) do with {:ok, %{status_code: 200, body: body}} <- @httpoison.post(query, file_data) do
File.rm!(path)
remote_file_name = String.split(body) |> List.first() remote_file_name = String.split(body) |> List.first()
public_url = "#{files}/#{remote_file_name}.#{extension}" public_url = "#{files}/#{remote_file_name}.#{extension}"
{:ok, {:url, public_url}} {:ok, {:url, public_url}}
else else
_ -> Pleroma.Uploaders.Local.put_file(name, uuid, path, content_type, opts) _ -> Pleroma.Uploaders.Local.put_file(upload)
end end
end end
end end

View File

@ -15,20 +15,18 @@ def get_file(file) do
])}} ])}}
end end
def put_file(name, uuid, path, content_type, _opts) do def put_file(upload = %Pleroma.Upload{}) do
config = Pleroma.Config.get([__MODULE__]) config = Pleroma.Config.get([__MODULE__])
bucket = Keyword.get(config, :bucket) bucket = Keyword.get(config, :bucket)
{:ok, file_data} = File.read(path) {:ok, file_data} = File.read(upload.tempfile)
File.rm!(path) s3_name = strict_encode(upload.path)
s3_name = "#{uuid}/#{strict_encode(name)}"
op = op =
ExAws.S3.put_object(bucket, s3_name, file_data, [ ExAws.S3.put_object(bucket, s3_name, file_data, [
{:acl, :public_read}, {:acl, :public_read},
{:content_type, content_type} {:content_type, upload.content_type}
]) ])
case ExAws.request(op) do case ExAws.request(op) do

View File

@ -5,10 +5,11 @@ def get_file(name) do
{:ok, {:url, Path.join([Pleroma.Config.get!([__MODULE__, :object_url]), name])}} {:ok, {:url, Path.join([Pleroma.Config.get!([__MODULE__, :object_url]), name])}}
end end
def put_file(name, uuid, tmp_path, content_type, _opts) do def put_file(upload) do
{:ok, file_data} = File.read(tmp_path) Pleroma.Uploaders.Swift.Client.upload_file(
remote_name = "#{uuid}/#{name}" upload.path,
File.read!(upload.tmpfile),
Pleroma.Uploaders.Swift.Client.upload_file(remote_name, file_data, content_type) upload.content_type
)
end end
end end

View File

@ -16,20 +16,25 @@ defmodule Pleroma.Uploaders.Uploader do
Returns: Returns:
* `:ok` which assumes `{:ok, upload.path}`
* `{:ok, spec}` where spec is: * `{:ok, spec}` where spec is:
* `{:file, filename :: String.t}` to handle reads with `get_file/1` (recommended) * `{:file, filename :: String.t}` to handle reads with `get_file/1` (recommended)
This allows to correctly proxy or redirect requests to the backend, while allowing to migrate backends without breaking any URL. This allows to correctly proxy or redirect requests to the backend, while allowing to migrate backends without breaking any URL.
* `{url, url :: String.t}` to bypass `get_file/2` and use the `url` directly in the activity.
* `{url, url :: String.t}` to bypass `get_file/2` and use the `url` directly in the activity.
* `{:error, String.t}` error information if the file failed to be saved to the backend. * `{:error, String.t}` error information if the file failed to be saved to the backend.
""" """
@callback put_file( @callback put_file(Pleroma.Upload.t()) ::
name :: String.t(), :ok | {:ok, {:file | :url, String.t()}} | {:error, String.t()}
uuid :: String.t(),
file :: File.t(), @spec put_file(module(), Pleroma.Upload.t()) ::
content_type :: String.t(), {:ok, {:file | :url, String.t()}} | {:error, String.t()}
options :: Map.t() def put_file(uploader, upload) do
) :: {:ok, {:file, String.t()} | {:url, String.t()}} | {:error, String.t()} case uploader.put_file(upload) do
:ok -> {:ok, {:file, upload.path}}
other -> other
end
end
end end

View File

@ -35,14 +35,6 @@ def create_app(conn, params) do
def update_credentials(%{assigns: %{user: user}} = conn, params) do def update_credentials(%{assigns: %{user: user}} = conn, params) do
original_user = user original_user = user
avatar_upload_limit =
Application.get_env(:pleroma, :instance)
|> Keyword.fetch(:avatar_upload_limit)
banner_upload_limit =
Application.get_env(:pleroma, :instance)
|> Keyword.fetch(:banner_upload_limit)
params = params =
if bio = params["note"] do if bio = params["note"] do
Map.put(params, "bio", bio) Map.put(params, "bio", bio)
@ -60,7 +52,7 @@ def update_credentials(%{assigns: %{user: user}} = conn, params) do
user = user =
if avatar = params["avatar"] do if avatar = params["avatar"] do
with %Plug.Upload{} <- avatar, with %Plug.Upload{} <- avatar,
{:ok, object} <- ActivityPub.upload(avatar, size_limit: avatar_upload_limit), {:ok, object} <- ActivityPub.upload(avatar, type: :avatar),
change = Ecto.Changeset.change(user, %{avatar: object.data}), change = Ecto.Changeset.change(user, %{avatar: object.data}),
{:ok, user} = User.update_and_set_cache(change) do {:ok, user} = User.update_and_set_cache(change) do
user user
@ -74,7 +66,7 @@ def update_credentials(%{assigns: %{user: user}} = conn, params) do
user = user =
if banner = params["header"] do if banner = params["header"] do
with %Plug.Upload{} <- banner, with %Plug.Upload{} <- banner,
{:ok, object} <- ActivityPub.upload(banner, size_limit: banner_upload_limit), {:ok, object} <- ActivityPub.upload(banner, type: :banner),
new_info <- Map.put(user.info, "banner", object.data), new_info <- Map.put(user.info, "banner", object.data),
change <- User.info_changeset(user, %{info: new_info}), change <- User.info_changeset(user, %{info: new_info}),
{:ok, user} <- User.update_and_set_cache(change) do {:ok, user} <- User.update_and_set_cache(change) do
@ -471,19 +463,12 @@ def update_media(%{assigns: %{user: _}} = conn, data) do
end end
def upload(%{assigns: %{user: _}} = conn, %{"file" => file} = data) do def upload(%{assigns: %{user: _}} = conn, %{"file" => file} = data) do
with {:ok, object} <- ActivityPub.upload(file) do with {:ok, object} <- ActivityPub.upload(file, description: Map.get(data, "description")) do
objdata = change = Object.change(object, %{data: object.data})
if Map.has_key?(data, "description") do
Map.put(object.data, "name", data["description"])
else
object.data
end
change = Object.change(object, %{data: objdata})
{:ok, object} = Repo.update(change) {:ok, object} = Repo.update(change)
objdata = objdata =
objdata object.data
|> Map.put("id", object.id) |> Map.put("id", object.id)
render(conn, StatusView, "attachment.json", %{attachment: objdata}) render(conn, StatusView, "attachment.json", %{attachment: objdata})

View File

@ -290,11 +290,7 @@ def register(conn, params) do
end end
def update_avatar(%{assigns: %{user: user}} = conn, params) do def update_avatar(%{assigns: %{user: user}} = conn, params) do
upload_limit = {:ok, object} = ActivityPub.upload(params, type: :avatar)
Application.get_env(:pleroma, :instance)
|> Keyword.fetch(:avatar_upload_limit)
{:ok, object} = ActivityPub.upload(params, size_limit: upload_limit)
change = Changeset.change(user, %{avatar: object.data}) change = Changeset.change(user, %{avatar: object.data})
{:ok, user} = User.update_and_set_cache(change) {:ok, user} = User.update_and_set_cache(change)
CommonAPI.update(user) CommonAPI.update(user)
@ -303,12 +299,7 @@ def update_avatar(%{assigns: %{user: user}} = conn, params) do
end end
def update_banner(%{assigns: %{user: user}} = conn, params) do def update_banner(%{assigns: %{user: user}} = conn, params) do
upload_limit = with {:ok, object} <- ActivityPub.upload(%{"img" => params["banner"]}, type: :banner),
Application.get_env(:pleroma, :instance)
|> Keyword.fetch(:banner_upload_limit)
with {:ok, object} <-
ActivityPub.upload(%{"img" => params["banner"]}, size_limit: upload_limit),
new_info <- Map.put(user.info, "banner", object.data), new_info <- Map.put(user.info, "banner", object.data),
change <- User.info_changeset(user, %{info: new_info}), change <- User.info_changeset(user, %{info: new_info}),
{:ok, user} <- User.update_and_set_cache(change) do {:ok, user} <- User.update_and_set_cache(change) do
@ -322,11 +313,7 @@ def update_banner(%{assigns: %{user: user}} = conn, params) do
end end
def update_background(%{assigns: %{user: user}} = conn, params) do def update_background(%{assigns: %{user: user}} = conn, params) do
upload_limit = with {:ok, object} <- ActivityPub.upload(params, type: :background),
Application.get_env(:pleroma, :instance)
|> Keyword.fetch(:background_upload_limit)
with {:ok, object} <- ActivityPub.upload(params, size_limit: upload_limit),
new_info <- Map.put(user.info, "background", object.data), new_info <- Map.put(user.info, "background", object.data),
change <- User.info_changeset(user, %{info: new_info}), change <- User.info_changeset(user, %{info: new_info}),
{:ok, _user} <- User.update_and_set_cache(change) do {:ok, _user} <- User.update_and_set_cache(change) do

View File

@ -5,16 +5,23 @@ defmodule Pleroma.UploadTest do
describe "Storing a file with the Local uploader" do describe "Storing a file with the Local uploader" do
setup do setup do
uploader = Pleroma.Config.get([Pleroma.Upload, :uploader]) uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
filters = Pleroma.Config.get([Pleroma.Upload, :filters])
unless uploader == Pleroma.Uploaders.Local || filters != [] do
Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
Pleroma.Config.put([Pleroma.Upload, :filters], [])
unless uploader == Pleroma.Uploaders.Local do
on_exit(fn -> on_exit(fn ->
Pleroma.Config.put([Pleroma.Upload, :uploader], uploader) Pleroma.Config.put([Pleroma.Upload, :uploader], uploader)
Pleroma.Config.put([Pleroma.Upload, :filters], filters)
end) end)
end end
:ok :ok
end end
OH - HELLO - EAL
test "returns a media url" do test "returns a media url" do
File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg") File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
@ -40,10 +47,11 @@ test "copies the file to the configured folder with deduping" do
filename: "an [image.jpg" filename: "an [image.jpg"
} }
{:ok, data} = Upload.store(file, dedupe: true) {:ok, data} = Upload.store(file, filters: [Pleroma.Upload.Filter.Dedupe])
assert data["name"] == assert List.first(data["url"])["href"] ==
"e7a6d0cf595bff76f14c9a98b6c199539559e8b844e02e51e5efcfd1f614a2df.jpeg" Pleroma.Web.base_url() <>
"/media/e7a6d0cf595bff76f14c9a98b6c199539559e8b844e02e51e5efcfd1f614a2df.jpg"
end end
test "copies the file to the configured folder without deduping" do test "copies the file to the configured folder without deduping" do
@ -55,7 +63,7 @@ test "copies the file to the configured folder without deduping" do
filename: "an [image.jpg" filename: "an [image.jpg"
} }
{:ok, data} = Upload.store(file, dedupe: false) {:ok, data} = Upload.store(file)
assert data["name"] == "an [image.jpg" assert data["name"] == "an [image.jpg"
end end
@ -68,7 +76,7 @@ test "fixes incorrect content type" do
filename: "an [image.jpg" filename: "an [image.jpg"
} }
{:ok, data} = Upload.store(file, dedupe: true) {:ok, data} = Upload.store(file, filters: [Pleroma.Upload.Filter.Dedupe])
assert hd(data["url"])["mediaType"] == "image/jpeg" assert hd(data["url"])["mediaType"] == "image/jpeg"
end end
@ -81,7 +89,7 @@ test "adds missing extension" do
filename: "an [image" filename: "an [image"
} }
{:ok, data} = Upload.store(file, dedupe: false) {:ok, data} = Upload.store(file)
assert data["name"] == "an [image.jpg" assert data["name"] == "an [image.jpg"
end end
@ -94,7 +102,7 @@ test "fixes incorrect file extension" do
filename: "an [image.blah" filename: "an [image.blah"
} }
{:ok, data} = Upload.store(file, dedupe: false) {:ok, data} = Upload.store(file)
assert data["name"] == "an [image.jpg" assert data["name"] == "an [image.jpg"
end end
@ -107,7 +115,7 @@ test "don't modify filename of an unknown type" do
filename: "test.txt" filename: "test.txt"
} }
{:ok, data} = Upload.store(file, dedupe: false) {:ok, data} = Upload.store(file)
assert data["name"] == "test.txt" assert data["name"] == "test.txt"
end end
end end