Merge branch 'feature/uploader_s3' into 'develop'
Uploader.S3 added support stream uploads See merge request pleroma/pleroma!1545
This commit is contained in:
commit
337edb3e50
@ -6,10 +6,12 @@ defmodule Pleroma.Uploaders.S3 do
|
|||||||
@behaviour Pleroma.Uploaders.Uploader
|
@behaviour Pleroma.Uploaders.Uploader
|
||||||
require Logger
|
require Logger
|
||||||
|
|
||||||
|
alias Pleroma.Config
|
||||||
|
|
||||||
# The file name is re-encoded with S3's constraints here to comply with previous
|
# The file name is re-encoded with S3's constraints here to comply with previous
|
||||||
# links with less strict filenames
|
# links with less strict filenames
|
||||||
def get_file(file) do
|
def get_file(file) do
|
||||||
config = Pleroma.Config.get([__MODULE__])
|
config = Config.get([__MODULE__])
|
||||||
bucket = Keyword.fetch!(config, :bucket)
|
bucket = Keyword.fetch!(config, :bucket)
|
||||||
|
|
||||||
bucket_with_namespace =
|
bucket_with_namespace =
|
||||||
@ -34,15 +36,15 @@ def get_file(file) do
|
|||||||
end
|
end
|
||||||
|
|
||||||
def put_file(%Pleroma.Upload{} = upload) do
|
def put_file(%Pleroma.Upload{} = upload) do
|
||||||
config = Pleroma.Config.get([__MODULE__])
|
config = Config.get([__MODULE__])
|
||||||
bucket = Keyword.get(config, :bucket)
|
bucket = Keyword.get(config, :bucket)
|
||||||
|
|
||||||
{:ok, file_data} = File.read(upload.tempfile)
|
|
||||||
|
|
||||||
s3_name = strict_encode(upload.path)
|
s3_name = strict_encode(upload.path)
|
||||||
|
|
||||||
op =
|
op =
|
||||||
ExAws.S3.put_object(bucket, s3_name, file_data, [
|
upload.tempfile
|
||||||
|
|> ExAws.S3.Upload.stream_file()
|
||||||
|
|> ExAws.S3.upload(bucket, s3_name, [
|
||||||
{:acl, :public_read},
|
{:acl, :public_read},
|
||||||
{:content_type, upload.content_type}
|
{:content_type, upload.content_type}
|
||||||
])
|
])
|
||||||
|
3
mix.exs
3
mix.exs
@ -114,8 +114,9 @@ defp deps do
|
|||||||
{:tesla, "~> 1.2"},
|
{:tesla, "~> 1.2"},
|
||||||
{:jason, "~> 1.0"},
|
{:jason, "~> 1.0"},
|
||||||
{:mogrify, "~> 0.6.1"},
|
{:mogrify, "~> 0.6.1"},
|
||||||
{:ex_aws, "~> 2.0"},
|
{:ex_aws, "~> 2.1"},
|
||||||
{:ex_aws_s3, "~> 2.0"},
|
{:ex_aws_s3, "~> 2.0"},
|
||||||
|
{:sweet_xml, "~> 0.6.6"},
|
||||||
{:earmark, "~> 1.3"},
|
{:earmark, "~> 1.3"},
|
||||||
{:bbcode, "~> 0.1.1"},
|
{:bbcode, "~> 0.1.1"},
|
||||||
{:ex_machina, "~> 2.3", only: :test},
|
{:ex_machina, "~> 2.3", only: :test},
|
||||||
|
1
mix.lock
1
mix.lock
@ -80,6 +80,7 @@
|
|||||||
"ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm"},
|
"ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm"},
|
||||||
"recon": {:git, "https://github.com/ferd/recon.git", "75d70c7c08926d2f24f1ee6de14ee50fe8a52763", [tag: "2.4.0"]},
|
"recon": {:git, "https://github.com/ferd/recon.git", "75d70c7c08926d2f24f1ee6de14ee50fe8a52763", [tag: "2.4.0"]},
|
||||||
"ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm"},
|
"ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm"},
|
||||||
|
"sweet_xml": {:hex, :sweet_xml, "0.6.6", "fc3e91ec5dd7c787b6195757fbcf0abc670cee1e4172687b45183032221b66b8", [:mix], [], "hexpm"},
|
||||||
"swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"},
|
"swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"},
|
||||||
"syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]},
|
"syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]},
|
||||||
"telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"},
|
"telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"},
|
||||||
|
90
test/uploaders/s3_test.exs
Normal file
90
test/uploaders/s3_test.exs
Normal file
@ -0,0 +1,90 @@
|
|||||||
|
# Pleroma: A lightweight social networking server
|
||||||
|
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
|
||||||
|
# SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
|
||||||
|
defmodule Pleroma.Uploaders.S3Test do
|
||||||
|
use Pleroma.DataCase
|
||||||
|
|
||||||
|
alias Pleroma.Config
|
||||||
|
alias Pleroma.Uploaders.S3
|
||||||
|
|
||||||
|
import Mock
|
||||||
|
import ExUnit.CaptureLog
|
||||||
|
|
||||||
|
setup do
|
||||||
|
config = Config.get([Pleroma.Uploaders.S3])
|
||||||
|
|
||||||
|
Config.put([Pleroma.Uploaders.S3],
|
||||||
|
bucket: "test_bucket",
|
||||||
|
public_endpoint: "https://s3.amazonaws.com"
|
||||||
|
)
|
||||||
|
|
||||||
|
on_exit(fn ->
|
||||||
|
Config.put([Pleroma.Uploaders.S3], config)
|
||||||
|
end)
|
||||||
|
|
||||||
|
:ok
|
||||||
|
end
|
||||||
|
|
||||||
|
describe "get_file/1" do
|
||||||
|
test "it returns path to local folder for files" do
|
||||||
|
assert S3.get_file("test_image.jpg") == {
|
||||||
|
:ok,
|
||||||
|
{:url, "https://s3.amazonaws.com/test_bucket/test_image.jpg"}
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
test "it returns path without bucket when truncated_namespace set to ''" do
|
||||||
|
Config.put([Pleroma.Uploaders.S3],
|
||||||
|
bucket: "test_bucket",
|
||||||
|
public_endpoint: "https://s3.amazonaws.com",
|
||||||
|
truncated_namespace: ""
|
||||||
|
)
|
||||||
|
|
||||||
|
assert S3.get_file("test_image.jpg") == {
|
||||||
|
:ok,
|
||||||
|
{:url, "https://s3.amazonaws.com/test_image.jpg"}
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
test "it returns path with bucket namespace when namespace is set" do
|
||||||
|
Config.put([Pleroma.Uploaders.S3],
|
||||||
|
bucket: "test_bucket",
|
||||||
|
public_endpoint: "https://s3.amazonaws.com",
|
||||||
|
bucket_namespace: "family"
|
||||||
|
)
|
||||||
|
|
||||||
|
assert S3.get_file("test_image.jpg") == {
|
||||||
|
:ok,
|
||||||
|
{:url, "https://s3.amazonaws.com/family:test_bucket/test_image.jpg"}
|
||||||
|
}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
describe "put_file/1" do
|
||||||
|
setup do
|
||||||
|
file_upload = %Pleroma.Upload{
|
||||||
|
name: "image-tet.jpg",
|
||||||
|
content_type: "image/jpg",
|
||||||
|
path: "test_folder/image-tet.jpg",
|
||||||
|
tempfile: Path.absname("test/fixtures/image_tmp.jpg")
|
||||||
|
}
|
||||||
|
|
||||||
|
[file_upload: file_upload]
|
||||||
|
end
|
||||||
|
|
||||||
|
test "save file", %{file_upload: file_upload} do
|
||||||
|
with_mock ExAws, request: fn _ -> {:ok, :ok} end do
|
||||||
|
assert S3.put_file(file_upload) == {:ok, {:file, "test_folder/image-tet.jpg"}}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
test "returns error", %{file_upload: file_upload} do
|
||||||
|
with_mock ExAws, request: fn _ -> {:error, "S3 Upload failed"} end do
|
||||||
|
assert capture_log(fn ->
|
||||||
|
assert S3.put_file(file_upload) == {:error, "S3 Upload failed"}
|
||||||
|
end) =~ "Elixir.Pleroma.Uploaders.S3: {:error, \"S3 Upload failed\"}"
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
Loading…
Reference in New Issue
Block a user