Merge branch 'feature/s3' into 'develop'

S3 support

Closes #65

See merge request pleroma/pleroma!303

config/config.exs

@@ -12,7 +12,9 @@ config :pleroma, Pleroma.Repo, types: Pleroma.PostgresTypes
 
 config :pleroma, Pleroma.Upload,
   uploads: "uploads",
-  strip_exif: false
+  strip_exif: false,
+  use_s3: false,
+  s3_bucket: nil
 
 config :pleroma, :emoji, shortcode_globs: ["/emoji/custom/**/*.png"]
 

lib/mix/tasks/sample_config.eex

@@ -24,3 +24,23 @@ config :pleroma, Pleroma.Repo,
   database: "pleroma_dev",
   hostname: "localhost",
   pool_size: 10
+
+# Configure S3 support if desired.
+# The public S3 endpoint is different depending on region and provider,
+# consult your S3 provider's documentation for details on what to use.
+#
+# config :pleroma, Pleroma.Upload,
+#   use_s3: true,
+#   bucket: "some-bucket",
+#   public_endpoint: "https://s3.amazonaws.com"
+#
+# Configure S3 credentials:
+# config :ex_aws, :s3,
+#   access_key_id: "xxxxxxxxxxxxx",
+#   secret_access_key: "yyyyyyyyyyyy",
+#   region: "us-east-1",
+#   scheme: "https://"
+#
+# For using third-party S3 clones like wasabi, also do:
+# config :ex_aws, :s3,
+#   host: "s3.wasabisys.com"

lib/pleroma/upload.ex

@@ -3,12 +3,17 @@ defmodule Pleroma.Upload do
   alias Pleroma.Web
 
   def store(%Plug.Upload{} = file, should_dedupe) do
+    settings = Application.get_env(:pleroma, Pleroma.Upload)
+    use_s3 = Keyword.fetch!(settings, :use_s3)
+
     content_type = get_content_type(file.path)
     uuid = get_uuid(file, should_dedupe)
     name = get_name(file, uuid, content_type, should_dedupe)
     upload_folder = get_upload_path(uuid, should_dedupe)
     url_path = get_url(name, uuid, should_dedupe)
 
+    strip_exif_data(content_type, file.path)
+
     File.mkdir_p!(upload_folder)
     result_file = Path.join(upload_folder, name)
 
@@ -18,7 +23,12 @@ defmodule Pleroma.Upload do
       File.cp!(file.path, result_file)
     end
 
-    strip_exif_data(content_type, result_file)
+    url_path =
+      if use_s3 do
+        put_s3_file(name, uuid, result_file, content_type)
+      else
+        url_path
+      end
 
     %{
       "type" => "Document",
@@ -33,7 +43,11 @@ defmodule Pleroma.Upload do
     }
   end
 
+  # XXX: does this code actually work?  i am skeptical.  --kaniini
   def store(%{"img" => "data:image/" <> image_data}, should_dedupe) do
+    settings = Application.get_env(:pleroma, Pleroma.Upload)
+    use_s3 = Keyword.fetch!(settings, :use_s3)
+
     parsed = Regex.named_captures(~r/(?<filetype>jpeg|png|gif);base64,(?<data>.*)/, image_data)
     data = Base.decode64!(parsed["data"], ignore: :whitespace)
     uuid = UUID.generate()
@@ -71,6 +85,13 @@ defmodule Pleroma.Upload do
 
     strip_exif_data(content_type, result_file)
 
+    url_path =
+      if use_s3 do
+        put_s3_file(name, uuid, result_file, content_type)
+      else
+        url_path
+      end
+
     %{
       "type" => "Image",
       "url" => [
@@ -203,4 +224,25 @@ defmodule Pleroma.Upload do
       _e -> "application/octet-stream"
     end
   end
+
+  defp put_s3_file(name, uuid, path, content_type) do
+    settings = Application.get_env(:pleroma, Pleroma.Upload)
+    bucket = Keyword.fetch!(settings, :bucket)
+    public_endpoint = Keyword.fetch!(settings, :public_endpoint)
+
+    {:ok, file_data} = File.read(path)
+
+    File.rm!(path)
+
+    s3_name = "#{uuid}/#{name}"
+
+    {:ok, result} =
+      ExAws.S3.put_object(bucket, s3_name, file_data, [
+        {:acl, :public_read},
+        {:content_type, content_type}
+      ])
+      |> ExAws.request()
+
+    "#{public_endpoint}/#{bucket}/#{s3_name}"
+  end
 end

mix.exs

@@ -46,6 +46,8 @@ defmodule Pleroma.Mixfile do
       {:httpoison, "~> 1.2.0"},
       {:jason, "~> 1.0"},
       {:mogrify, "~> 0.6.1"},
+      {:ex_aws, "~> 2.0"},
+      {:ex_aws_s3, "~> 2.0"},
       {:ex_machina, "~> 2.2", only: :test},
       {:credo, "~> 0.9.3", only: [:dev, :test]},
       {:mock, "~> 0.3.1", only: :test}

mix.lock

@@ -12,6 +12,8 @@
   "decimal": {:hex, :decimal, "1.5.0", "b0433a36d0e2430e3d50291b1c65f53c37d56f83665b43d79963684865beab68", [:mix], [], "hexpm"},
   "ecto": {:hex, :ecto, "2.2.10", "e7366dc82f48f8dd78fcbf3ab50985ceeb11cb3dc93435147c6e13f2cda0992e", [:mix], [{:db_connection, "~> 1.1", [hex: :db_connection, repo: "hexpm", optional: true]}, {:decimal, "~> 1.2", [hex: :decimal, repo: "hexpm", optional: false]}, {:mariaex, "~> 0.8.0", [hex: :mariaex, repo: "hexpm", optional: true]}, {:poison, "~> 2.2 or ~> 3.0", [hex: :poison, repo: "hexpm", optional: true]}, {:poolboy, "~> 1.5", [hex: :poolboy, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.13.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:sbroker, "~> 1.0", [hex: :sbroker, repo: "hexpm", optional: true]}], "hexpm"},
   "eternal": {:hex, :eternal, "1.2.0", "e2a6b6ce3b8c248f7dc31451aefca57e3bdf0e48d73ae5043229380a67614c41", [:mix], [], "hexpm"},
+  "ex_aws": {:hex, :ex_aws, "2.1.0", "b92651527d6c09c479f9013caa9c7331f19cba38a650590d82ebf2c6c16a1d8a", [:mix], [{:configparser_ex, "~> 2.0", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "1.6.3 or 1.6.5 or 1.7.1 or 1.8.6 or ~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8", [hex: :jsx, repo: "hexpm", optional: true]}, {:poison, ">= 1.2.0", [hex: :poison, repo: "hexpm", optional: true]}, {:sweet_xml, "~> 0.6", [hex: :sweet_xml, repo: "hexpm", optional: true]}, {:xml_builder, "~> 0.1.0", [hex: :xml_builder, repo: "hexpm", optional: true]}], "hexpm"},
+  "ex_aws_s3": {:hex, :ex_aws_s3, "2.0.1", "9e09366e77f25d3d88c5393824e613344631be8db0d1839faca49686e99b6704", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:sweet_xml, ">= 0.0.0", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm"},
   "ex_machina": {:hex, :ex_machina, "2.2.0", "fec496331e04fc2db2a1a24fe317c12c0c4a50d2beb8ebb3531ed1f0d84be0ed", [:mix], [{:ecto, "~> 2.1", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"},
   "gettext": {:hex, :gettext, "0.15.0", "40a2b8ce33a80ced7727e36768499fc9286881c43ebafccae6bab731e2b2b8ce", [:mix], [], "hexpm"},
   "hackney": {:hex, :hackney, "1.13.0", "24edc8cd2b28e1c652593833862435c80661834f6c9344e84b6a2255e7aeef03", [:rebar3], [{:certifi, "2.3.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "5.1.2", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "1.0.2", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"},