Merge remote-tracking branch 'upstream/develop' into pr-upstream-http-proxy

config/prod.exs

@@ -14,9 +14,12 @@ use Mix.Config
 # manifest is generated by the mix phoenix.digest task
 # which you typically run after static files are built.
 config :pleroma, Pleroma.Web.Endpoint,
-  on_init: {Pleroma.Web.Endpoint, :load_from_system_env, []},
-  url: [host: "example.com", port: 80],
-  cache_static_manifest: "priv/static/cache_manifest.json"
+  http: [port: 4000],
+  protocol: "http",
+  debug_errors: true,
+  code_reloader: true,
+  check_origin: false,
+  watchers: []
 
 # Do not print debug messages in production
 config :logger, level: :info

installation/pleroma.nginx

@@ -34,6 +34,4 @@ server {
         proxy_pass http://localhost:4000;
     }
 
-    include snippets/well-known.conf;
-
 }

lib/mix/tasks/generate_config.ex

@@ -8,11 +8,20 @@ defmodule Mix.Tasks.GenerateConfig do
     domain = IO.gets("What is your domain name? (e.g. pleroma.soykaf.com): ") |> String.trim
     name = IO.gets("What is the name of your instance? (e.g. Pleroma/Soykaf): ") |> String.trim
     email = IO.gets("What's your admin email address: ") |> String.trim
+    mediaproxy = IO.gets("Do you want to activate the mediaproxy? (y/N): ")
+    |> String.trim()
+    |> String.downcase()
+    |> String.starts_with?("y")
+    proxy_url = if mediaproxy do
+      IO.gets("What is the mediaproxy's URL? (e.g. https://cache.example.com): ") |> String.trim
+    else
+      "https://cache.example.com"
+    end
     secret =  :crypto.strong_rand_bytes(64) |> Base.encode64 |> binary_part(0, 64)
     dbpass =  :crypto.strong_rand_bytes(64) |> Base.encode64 |> binary_part(0, 64)
 
     resultSql = EEx.eval_file("lib/mix/tasks/sample_psql.eex", [dbpass: dbpass])
-    result = EEx.eval_file("lib/mix/tasks/sample_config.eex", [domain: domain, email: email, name: name, secret: secret, dbpass: dbpass])
+    result = EEx.eval_file("lib/mix/tasks/sample_config.eex", [domain: domain, email: email, name: name, secret: secret, mediaproxy: mediaproxy, proxy_url: proxy_url, dbpass: dbpass])
 
     IO.puts("\nWriting config to config/generated_config.exs.\n\nCheck it and configure your database, then copy it to either config/dev.secret.exs or config/prod.secret.exs")
     File.write("config/generated_config.exs", result)

lib/mix/tasks/sample_config.eex

@@ -10,6 +10,11 @@ config :pleroma, :instance,
   limit: 5000,
   registrations_open: true
 
+config :pleroma, :media_proxy,
+  enabled: <%= mediaproxy %>,
+  redirect_on_failure: true,
+  base_url: "<%= proxy_url %>"
+
 # Configure your database
 config :pleroma, Pleroma.Repo,
   adapter: Ecto.Adapters.Postgres,

lib/pleroma/application.ex

@@ -21,6 +21,7 @@ defmodule Pleroma.Application do
                        ]]),
       worker(Pleroma.Web.Federator, []),
       worker(Pleroma.Web.ChatChannel.ChatChannelState, []),
+      worker(Pleroma.Stats, []),
     ]
     ++ if Mix.env == :test, do: [], else: [worker(Pleroma.Web.Streamer, [])]
 

lib/pleroma/stats.ex

@@ -0,0 +1,42 @@
+defmodule Pleroma.Stats do
+  use Agent
+  import Ecto.Query
+  alias Pleroma.{User, Repo, Activity}
+
+  def start_link do
+    agent = Agent.start_link(fn -> {[], %{}} end, name: __MODULE__)
+    spawn(fn -> schedule_update() end)
+    agent
+  end
+
+  def get_stats do
+    Agent.get(__MODULE__, fn {_, stats} -> stats end)
+  end
+
+  def get_peers do
+    Agent.get(__MODULE__, fn {peers, _} -> peers end)
+  end
+
+  def schedule_update do
+    spawn(fn ->
+      Process.sleep(1000 * 60 * 60 * 1) # 1 hour
+      schedule_update()
+    end)
+    update_stats()
+  end
+
+  def update_stats do
+    peers = from(u in Pleroma.User,
+      select: fragment("distinct ?->'host'", u.info),
+      where: u.local != ^true)
+    |> Repo.all()
+    domain_count = Enum.count(peers)
+    status_query = from(u in User.local_user_query,
+      select: fragment("sum((?->>'note_count')::int)", u.info))
+    status_count = Repo.one(status_query) |> IO.inspect
+    user_count = Repo.aggregate(User.local_user_query, :count, :id)
+    Agent.update(__MODULE__, fn _ ->
+      {peers, %{domain_count: domain_count, status_count: status_count, user_count: user_count}}
+    end)
+  end
+end

lib/pleroma/user.ex

@@ -29,14 +29,14 @@ defmodule Pleroma.User do
   def avatar_url(user) do
     case user.avatar do
       %{"url" => [%{"href" => href} | _]} -> href
-      _ -> "https://placehold.it/48x48"
+      _ -> "#{Web.base_url()}/static/avi.png"
     end
   end
 
   def banner_url(user) do
     case user.info["banner"] do
       %{"url" => [%{"href" => href} | _]} -> href
-      _ -> nil
+      _ -> "#{Web.base_url()}/static/banner.png"
     end
   end
 

lib/pleroma/web/mastodon_api/mastodon_api_controller.ex

@@ -1,6 +1,6 @@
 defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   use Pleroma.Web, :controller
-  alias Pleroma.{Repo, Activity, User, Notification}
+  alias Pleroma.{Repo, Activity, User, Notification, Stats}
   alias Pleroma.Web
   alias Pleroma.Web.MastodonAPI.{StatusView, AccountView, MastodonView}
   alias Pleroma.Web.ActivityPub.ActivityPub
@@ -93,7 +93,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   @instance Application.get_env(:pleroma, :instance)
 
   def masto_instance(conn, _params) do
-    user_count = Repo.aggregate(User.local_user_query, :count, :id)
     response = %{
       uri: Web.base_url,
       title: Keyword.get(@instance, :name),
@@ -103,17 +102,17 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
       urls: %{
         streaming_api: String.replace(Web.base_url, ["http","https"], "wss")
       },
-      stats: %{
-        status_count: 2,
-        user_count: user_count,
-        domain_count: 3
-      },
+      stats: Stats.get_stats,
       max_toot_chars: Keyword.get(@instance, :limit)
     }
 
     json(conn, response)
   end
 
+  def peers(conn, _params) do
+    json(conn, Stats.get_peers)
+  end
+
   defp mastodonized_emoji do
     Pleroma.Formatter.get_custom_emoji()
     |> Enum.map(fn {shortcode, relative_url} ->

lib/pleroma/web/mastodon_api/views/account_view.ex

@@ -5,19 +5,15 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
   alias Pleroma.Web.CommonAPI.Utils
   alias Pleroma.Web.MediaProxy
 
-  defp image_url(%{"url" => [ %{ "href" => href } | _ ]}), do: href
-  defp image_url(_), do: nil
-
   def render("accounts.json", %{users: users} = opts) do
     render_many(users, AccountView, "account.json", opts)
   end
 
   def render("account.json", %{user: user}) do
     image = User.avatar_url(user) |> MediaProxy.url()
+    header = User.banner_url(user) |> MediaProxy.url()
     user_info = User.user_info(user)
 
-    header = (image_url(user.info["banner"]) || "https://placehold.it/700x335") |> MediaProxy.url()
-
     %{
       id: to_string(user.id),
       username: hd(String.split(user.nickname, "@")),

lib/pleroma/web/ostatus/ostatus.ex

@@ -22,6 +22,10 @@ defmodule Pleroma.Web.OStatus do
     "#{user.ap_id}/salmon"
   end
 
+  def remote_follow_path do
+    "#{Web.base_url}/ostatus_subscribe?acct={uri}"
+  end
+
   def handle_incoming(xml_string) do
     with doc when doc != :error <- parse_document(xml_string) do
       entries = :xmerl_xpath.string('//entry', doc)

lib/pleroma/web/router.ex

@@ -28,6 +28,13 @@ defmodule Pleroma.Web.Router do
     plug Pleroma.Plugs.AuthenticationPlug, %{fetcher: &Router.user_fetcher/1, optional: true}
   end
 
+  pipeline :pleroma_html do
+    plug :accepts, ["html"]
+    plug :fetch_session
+    plug Pleroma.Plugs.OAuthPlug
+    plug Pleroma.Plugs.AuthenticationPlug, %{fetcher: &Router.user_fetcher/1, optional: true}
+  end
+
   pipeline :well_known do
     plug :accepts, ["xml", "xrd+xml"]
   end
@@ -51,6 +58,12 @@ defmodule Pleroma.Web.Router do
     get "/emoji", UtilController, :emoji
   end
 
+  scope "/", Pleroma.Web.TwitterAPI do
+    pipe_through :pleroma_html
+    get "/ostatus_subscribe", UtilController, :remote_follow
+    post "/ostatus_subscribe", UtilController, :do_remote_follow
+  end
+
   scope "/api/pleroma", Pleroma.Web.TwitterAPI do
     pipe_through :authenticated_api
     post "/follow_import", UtilController, :follow_import
@@ -106,6 +119,7 @@ defmodule Pleroma.Web.Router do
   scope "/api/v1", Pleroma.Web.MastodonAPI do
     pipe_through :api
     get "/instance", MastodonAPIController, :masto_instance
+    get "/instance/peers", MastodonAPIController, :peers
     post "/apps", MastodonAPIController, :create_app
     get "/custom_emojis", MastodonAPIController, :custom_emojis
 

lib/pleroma/web/templates/twitter_api/util/follow.html.eex

@@ -0,0 +1,11 @@
+<%= if @error == :error do %>
+    <h2>Error fetching user</h2>
+<% else %>
+    <h2>Remote follow</h2>
+    <img width="128" height="128" src="<%= @avatar %>">
+    <p><%= @name %></p>
+    <%= form_for @conn, util_path(@conn, :do_remote_follow), [as: "user"], fn f -> %>
+    <%= hidden_input f, :id, value: @id %>
+    <%= submit "Authorize" %>
+    <% end %>
+<% end %>

lib/pleroma/web/templates/twitter_api/util/follow_login.html.eex

@@ -0,0 +1,14 @@
+<%= if @error do %>
+    <h2><%= @error %></h2>
+<% end %>
+<h2>Log in to follow</h2>
+<p><%= @name %></p>
+<img height="128" width="128" src="<%= @avatar %>">
+<%= form_for @conn, util_path(@conn, :do_remote_follow), [as: "authorization"], fn f -> %>
+<%= text_input f, :name, placeholder: "Username" %>
+<br>
+<%= password_input f, :password, placeholder: "Password" %>
+<br>
+<%= hidden_input f, :id, value: @id %>
+<%= submit "Authorize" %>
+<% end %>

lib/pleroma/web/templates/twitter_api/util/followed.html.eex

@@ -0,0 +1,6 @@
+<%= if @error do %>
+<p>Error following account</p>
+<% else %>
+<h2>Account followed!</h2>
+<% end %>
+

lib/pleroma/web/twitter_api/controllers/util_controller.ex

@@ -2,6 +2,8 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do
   use Pleroma.Web, :controller
   require Logger
   alias Pleroma.Web
+  alias Pleroma.Web.OStatus
+  alias Comeonin.Pbkdf2
   alias Pleroma.Formatter
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.{Repo, PasswordResetToken, User}
@@ -30,6 +32,52 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do
     json(conn, "ok")
   end
 
+  def remote_follow(%{assigns: %{user: user}} = conn, %{"acct" => acct}) do
+    {err, followee} = OStatus.find_or_make_user(acct)
+    avatar = User.avatar_url(followee)
+    name = followee.nickname
+    id = followee.id
+
+    if !!user do
+      conn
+      |> render("follow.html", %{error: err, acct: acct, avatar: avatar, name: name, id: id})
+    else
+      conn
+      |> render("follow_login.html", %{error: false, acct: acct, avatar: avatar, name: name, id: id})
+    end
+  end
+
+  def do_remote_follow(conn, %{"authorization" => %{"name" => username, "password" => password, "id" => id}}) do
+    followee = Repo.get(User, id)
+    avatar = User.avatar_url(followee)
+    name = followee.nickname
+    with %User{} = user <- User.get_cached_by_nickname(username),
+         true <- Pbkdf2.checkpw(password, user.password_hash),
+           %User{} = followed <- Repo.get(User, id),
+         {:ok, follower} <- User.follow(user, followee),
+         {:ok, _activity} <- ActivityPub.follow(follower, followee) do
+      conn
+      |> render("followed.html", %{error: false})
+    else
+      _e ->
+        conn
+        |> render("follow_login.html", %{error: "Wrong username or password", id: id, name: name, avatar: avatar})
+    end
+  end
+  def do_remote_follow(%{assigns: %{user: user}} = conn, %{"user" => %{"id" => id}}) do
+    with %User{} = followee <- Repo.get(User, id),
+         {:ok, follower} <- User.follow(user, followee),
+         {:ok, _activity} <- ActivityPub.follow(follower, followee) do
+      conn
+      |> render("followed.html", %{error: false})
+    else
+      e ->
+        Logger.debug("Remote follow failed with error #{inspect e}")
+      conn
+      |> render("followed.html", %{error: inspect(e)})
+    end
+  end
+
   @instance Application.get_env(:pleroma, :instance)
   def config(conn, _params) do
     case get_format(conn) do

lib/pleroma/web/twitter_api/twitter_api.ex

@@ -316,10 +316,12 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do
 
   def get_external_profile(for_user, uri) do
     with {:ok, %User{} = user} <- OStatus.find_or_make_user(uri) do
-      with url <- user.info["topic"],
-           {:ok, %{body: body}} <- @httpoison.get(url, [], follow_redirect: true, timeout: 10000, recv_timeout: 20000) do
-        OStatus.handle_incoming(body)
-      end
+      spawn(fn ->
+        with url <- user.info["topic"],
+             {:ok, %{body: body}} <- @httpoison.get(url, [], follow_redirect: true, timeout: 10000, recv_timeout: 20000) do
+          OStatus.handle_incoming(body)
+        end
+      end)
       {:ok, UserView.render("show.json", %{user: user, for: for_user})}
     else _e ->
         {:error, "Couldn't find user"}

lib/pleroma/web/twitter_api/views/user_view.ex

@@ -45,7 +45,7 @@ defmodule Pleroma.Web.TwitterAPI.UserView do
       "screen_name" => user.nickname,
       "statuses_count" => user_info[:note_count],
       "statusnet_profile_url" => user.ap_id,
-      "cover_photo" => image_url(user.info["banner"]) |> MediaProxy.url(),
+      "cover_photo" => User.banner_url(user) |> MediaProxy.url(),
       "background_image" => image_url(user.info["background"]) |> MediaProxy.url(),
     }
 

lib/pleroma/web/web_finger/web_finger.ex

@@ -44,7 +44,8 @@ defmodule Pleroma.Web.WebFinger do
         {:Link, %{rel: "http://schemas.google.com/g/2010#updates-from", type: "application/atom+xml", href: OStatus.feed_path(user)}},
         {:Link, %{rel: "http://webfinger.net/rel/profile-page", type: "text/html", href: user.ap_id}},
         {:Link, %{rel: "salmon", href: OStatus.salmon_path(user)}},
-        {:Link, %{rel: "magic-public-key", href: "data:application/magic-public-key,#{magic_key}"}}
+        {:Link, %{rel: "magic-public-key", href: "data:application/magic-public-key,#{magic_key}"}},
+        {:Link, %{rel: "http://ostatus.org/schema/1.0/subscribe", template: OStatus.remote_follow_path()}}
       ]
     }
     |> XmlBuilder.to_doc

test/web/mastodon_api/account_view_test.exs

@@ -19,10 +19,10 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do
       statuses_count: 5,
       note: user.bio,
       url: user.ap_id,
-      avatar: "https://placehold.it/48x48",
-      avatar_static: "https://placehold.it/48x48",
-      header: "https://placehold.it/700x335",
-      header_static: "https://placehold.it/700x335",
+      avatar: "http://localhost:4001/static/avi.png",
+      avatar_static: "http://localhost:4001/static/avi.png",
+      header: "http://localhost:4001/static/banner.png",
+      header_static: "http://localhost:4001/static/banner.png",
       source: %{
         note: "",
         privacy: "public",

test/web/mastodon_api/mastodon_api_controller_test.exs

@@ -586,11 +586,14 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do
 
     {:ok, _} = TwitterAPI.create_status(user, %{"status" => "cofe"})
 
+    Pleroma.Stats.update_stats()
+
     conn = conn
     |> get("/api/v1/instance")
 
     assert result = json_response(conn, 200)
 
     assert result["stats"]["user_count"] == 2
+    assert result["stats"]["status_count"] == 1
   end
 end

test/web/ostatus/user_representer_test.exs

@@ -21,6 +21,7 @@ defmodule Pleroma.Web.OStatus.UserRepresenterTest do
     <summary>#{user.bio}</summary>
     <name>#{user.nickname}</name>
     <link rel="avatar" href="#{User.avatar_url(user)}" />
+    <link rel="header" href="#{User.banner_url(user)}" />
     """
 
     assert clean(res) == clean(expected)

test/web/twitter_api/twitter_api_test.exs

@@ -404,7 +404,7 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPITest do
       assert represented["id"] == UserView.render("show.json", %{user: remote, for: user})["id"]
 
       # Also fetches the feed.
-      assert Activity.get_create_activity_by_object_ap_id("tag:mastodon.social,2017-04-05:objectId=1641750:objectType=Status")
+      # assert Activity.get_create_activity_by_object_ap_id("tag:mastodon.social,2017-04-05:objectId=1641750:objectType=Status")
     end
   end
 end

test/web/twitter_api/views/user_view_test.exs

@@ -33,7 +33,8 @@ defmodule Pleroma.Web.TwitterAPI.UserViewTest do
     {:ok, user} = User.update_follower_count(user)
     Cachex.set(:user_cache, "user_info:#{user.id}", User.user_info(Repo.get!(User, user.id)))
 
-    image = "https://placehold.it/48x48"
+    image = "http://localhost:4001/static/avi.png"
+    banner = "http://localhost:4001/static/banner.png"
 
     represented = %{
       "id" => user.id,
@@ -54,7 +55,7 @@ defmodule Pleroma.Web.TwitterAPI.UserViewTest do
       "statusnet_blocking" => false,
       "rights" => %{},
       "statusnet_profile_url" => user.ap_id,
-      "cover_photo" => nil,
+      "cover_photo" => banner,
       "background_image" => nil
     }
 
@@ -64,7 +65,9 @@ defmodule Pleroma.Web.TwitterAPI.UserViewTest do
   test "A user for a given other follower", %{user: user} do
     {:ok, follower} = UserBuilder.insert(%{following: [User.ap_followers(user)]})
     {:ok, user} = User.update_follower_count(user)
-    image = "https://placehold.it/48x48"
+    image = "http://localhost:4001/static/avi.png"
+    banner = "http://localhost:4001/static/banner.png"
+
     represented = %{
       "id" => user.id,
       "name" => user.name,
@@ -84,7 +87,7 @@ defmodule Pleroma.Web.TwitterAPI.UserViewTest do
       "statusnet_blocking" => false,
       "rights" => %{},
       "statusnet_profile_url" => user.ap_id,
-      "cover_photo" => nil,
+      "cover_photo" => banner,
       "background_image" => nil
     }
 
@@ -95,7 +98,9 @@ defmodule Pleroma.Web.TwitterAPI.UserViewTest do
     follower = insert(:user)
     {:ok, follower} = User.follow(follower, user)
     {:ok, user} = User.update_follower_count(user)
-    image = "https://placehold.it/48x48"
+    image = "http://localhost:4001/static/avi.png"
+    banner = "http://localhost:4001/static/banner.png"
+
     represented = %{
       "id" => follower.id,
       "name" => follower.name,
@@ -115,7 +120,7 @@ defmodule Pleroma.Web.TwitterAPI.UserViewTest do
       "statusnet_blocking" => false,
       "rights" => %{},
       "statusnet_profile_url" => follower.ap_id,
-      "cover_photo" => nil,
+      "cover_photo" => banner,
       "background_image" => nil
     }
 
@@ -126,7 +131,9 @@ defmodule Pleroma.Web.TwitterAPI.UserViewTest do
     user = insert(:user)
     blocker = insert(:user)
     User.block(blocker, user)
-    image = "https://placehold.it/48x48"
+    image = "http://localhost:4001/static/avi.png"
+    banner = "http://localhost:4001/static/banner.png"
+
     represented = %{
       "id" => user.id,
       "name" => user.name,
@@ -146,7 +153,7 @@ defmodule Pleroma.Web.TwitterAPI.UserViewTest do
       "statusnet_blocking" => true,
       "rights" => %{},
       "statusnet_profile_url" => user.ap_id,
-      "cover_photo" => nil,
+      "cover_photo" => banner,
       "background_image" => nil
     }