From 1097ce6d9f06a7552652c5990cee12e7b7b3cc59 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Thu, 21 Feb 2019 18:55:19 +0300
Subject: [PATCH 01/68] Auth customization support.
 OAuthController#create_authorization user retrieval / creation, errors
 handling, template & layout selection.

---
 lib/pleroma/web/oauth.ex                      |  8 +++++++
 lib/pleroma/web/oauth/authenticator.ex        | 22 +++++++++++++++++++
 .../web/oauth/authenticator_adapter.ex        |  7 ++++++
 lib/pleroma/web/oauth/oauth_controller.ex     | 17 +++++++-------
 lib/pleroma/web/web.ex                        |  2 ++
 5 files changed, 48 insertions(+), 8 deletions(-)
 create mode 100644 lib/pleroma/web/oauth/authenticator.ex
 create mode 100644 lib/pleroma/web/oauth/authenticator_adapter.ex

diff --git a/lib/pleroma/web/oauth.ex b/lib/pleroma/web/oauth.ex
index d2835a0ba..f3bac33c8 100644
--- a/lib/pleroma/web/oauth.ex
+++ b/lib/pleroma/web/oauth.ex
@@ -3,6 +3,14 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.OAuth do
+  @authenticator Application.get_env(
+                   :pleroma,
+                   Pleroma.Web.AuthenticatorAdapter,
+                   Pleroma.Web.Authenticator
+                 )
+
+  def authenticator, do: @authenticator
+
   def parse_scopes(scopes, _default) when is_list(scopes) do
     Enum.filter(scopes, &(&1 not in [nil, ""]))
   end
diff --git a/lib/pleroma/web/oauth/authenticator.ex b/lib/pleroma/web/oauth/authenticator.ex
new file mode 100644
index 000000000..86bbc41f0
--- /dev/null
+++ b/lib/pleroma/web/oauth/authenticator.ex
@@ -0,0 +1,22 @@
+defmodule Pleroma.Web.Authenticator do
+  alias Pleroma.User
+  alias Comeonin.Pbkdf2
+
+  @behaviour Pleroma.Web.AuthenticatorAdapter
+
+  def get_user(%Plug.Conn{} = conn) do
+    %{"authorization" => %{"name" => name, "password" => password}} = conn.params
+
+    with {_, %User{} = user} <- {:user, User.get_by_nickname_or_email(name)},
+         {_, true} <- {:checkpw, Pbkdf2.checkpw(password, user.password_hash)} do
+      {:ok, user}
+    else
+      error ->
+        {:error, error}
+    end
+  end
+
+  def handle_error(%Plug.Conn{} = _conn, error) do
+    error
+  end
+end
diff --git a/lib/pleroma/web/oauth/authenticator_adapter.ex b/lib/pleroma/web/oauth/authenticator_adapter.ex
new file mode 100644
index 000000000..282963b1c
--- /dev/null
+++ b/lib/pleroma/web/oauth/authenticator_adapter.ex
@@ -0,0 +1,7 @@
+defmodule Pleroma.Web.AuthenticatorAdapter do
+  alias Pleroma.User
+
+  @callback get_user(Plug.Conn.t()) :: {:ok, User.t()} | {:error, any()}
+
+  @callback handle_error(Plug.Conn.t(), any()) :: any()
+end
diff --git a/lib/pleroma/web/oauth/oauth_controller.ex b/lib/pleroma/web/oauth/oauth_controller.ex
index 7c1a3adbd..abe6fd2f2 100644
--- a/lib/pleroma/web/oauth/oauth_controller.ex
+++ b/lib/pleroma/web/oauth/oauth_controller.ex
@@ -5,6 +5,7 @@
 defmodule Pleroma.Web.OAuth.OAuthController do
   use Pleroma.Web, :controller
 
+  alias Pleroma.Web.OAuth
   alias Pleroma.Web.OAuth.Authorization
   alias Pleroma.Web.OAuth.Token
   alias Pleroma.Web.OAuth.App
@@ -24,27 +25,27 @@ defmodule Pleroma.Web.OAuth.OAuthController do
     available_scopes = (app && app.scopes) || []
     scopes = oauth_scopes(params, nil) || available_scopes
 
-    render(conn, "show.html", %{
+    template = Application.get_env(:pleroma, :auth_template, "show.html")
+
+    render(conn, template, %{
       response_type: params["response_type"],
       client_id: params["client_id"],
       available_scopes: available_scopes,
       scopes: scopes,
       redirect_uri: params["redirect_uri"],
-      state: params["state"]
+      state: params["state"],
+      params: params
     })
   end
 
   def create_authorization(conn, %{
         "authorization" =>
           %{
-            "name" => name,
-            "password" => password,
             "client_id" => client_id,
             "redirect_uri" => redirect_uri
           } = auth_params
       }) do
-    with %User{} = user <- User.get_by_nickname_or_email(name),
-         true <- Pbkdf2.checkpw(password, user.password_hash),
+    with {_, {:ok, %User{} = user}} <- {:get_user, OAuth.authenticator().get_user(conn)},
          %App{} = app <- Repo.get_by(App, client_id: client_id),
          true <- redirect_uri in String.split(app.redirect_uris),
          scopes <- oauth_scopes(auth_params, []),
@@ -53,9 +54,9 @@ defmodule Pleroma.Web.OAuth.OAuthController do
          {:missing_scopes, false} <- {:missing_scopes, scopes == []},
          {:auth_active, true} <- {:auth_active, User.auth_active?(user)},
          {:ok, auth} <- Authorization.create_authorization(app, user, scopes) do
-      # Special case: Local MastodonFE.
       redirect_uri =
         if redirect_uri == "." do
+          # Special case: Local MastodonFE
           mastodon_api_url(conn, :login)
         else
           redirect_uri
@@ -97,7 +98,7 @@ defmodule Pleroma.Web.OAuth.OAuthController do
         |> authorize(auth_params)
 
       error ->
-        error
+        OAuth.authenticator().handle_error(conn, error)
     end
   end
 
diff --git a/lib/pleroma/web/web.ex b/lib/pleroma/web/web.ex
index 853aa2a87..4f618743d 100644
--- a/lib/pleroma/web/web.ex
+++ b/lib/pleroma/web/web.ex
@@ -26,6 +26,8 @@ defmodule Pleroma.Web do
       import Plug.Conn
       import Pleroma.Web.Gettext
       import Pleroma.Web.Router.Helpers
+
+      plug(:put_layout, Application.get_env(:pleroma, :app_template, "app.html"))
     end
   end
 

From afddce45b3d4d9ea61620c941838a372ca225825 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Fri, 22 Feb 2019 11:10:17 +0300
Subject: [PATCH 02/68] Minor setting name adjustment (:app_template ->
 :app_layout).

---
 lib/pleroma/web/web.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/web.ex b/lib/pleroma/web/web.ex
index 4f618743d..4bf07a6ef 100644
--- a/lib/pleroma/web/web.ex
+++ b/lib/pleroma/web/web.ex
@@ -27,7 +27,7 @@ defmodule Pleroma.Web do
       import Pleroma.Web.Gettext
       import Pleroma.Web.Router.Helpers
 
-      plug(:put_layout, Application.get_env(:pleroma, :app_template, "app.html"))
+      plug(:put_layout, Application.get_env(:pleroma, :app_layout, "app.html"))
     end
   end
 

From b227ccab827471359f539d23a14413e70ad67e33 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Fri, 22 Feb 2019 14:00:18 +0300
Subject: [PATCH 03/68] Fixed `scopes` of apps / authorizations / tokens from
 apps initially created with space-delimited `scope`.

---
 ...20190222104808_data_migration_normalize_scopes.exs | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 priv/repo/migrations/20190222104808_data_migration_normalize_scopes.exs

diff --git a/priv/repo/migrations/20190222104808_data_migration_normalize_scopes.exs b/priv/repo/migrations/20190222104808_data_migration_normalize_scopes.exs
new file mode 100644
index 000000000..d44e5096b
--- /dev/null
+++ b/priv/repo/migrations/20190222104808_data_migration_normalize_scopes.exs
@@ -0,0 +1,11 @@
+defmodule Pleroma.Repo.Migrations.DataMigrationNormalizeScopes do
+  use Ecto.Migration
+
+  def up do
+    for t <- [:apps, :oauth_authorizations, :oauth_tokens] do
+      execute "UPDATE #{t} SET scopes = string_to_array(array_to_string(scopes, ' '), ' ');"
+    end
+  end
+
+  def down, do: :noop
+end

From 62296f5a251e376bed5b234a66b20226dbd58419 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 22 Feb 2019 12:02:51 +0100
Subject: [PATCH 04/68] Fix private post card handling.

---
 lib/pleroma/web/activity_pub/activity_pub.ex         |  2 +-
 .../web/mastodon_api/mastodon_api_controller.ex      |  4 ++--
 .../mastodon_api/mastodon_api_controller_test.exs    | 12 ++++++++++++
 3 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 7e153f396..fb0b7b68e 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -943,7 +943,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   def visible_for_user?(activity, user) do
     x = [user.ap_id | user.following]
-    y = activity.data["to"] ++ (activity.data["cc"] || [])
+    y = [activity.actor] ++ activity.data["to"] ++ (activity.data["cc"] || [])
     visible_for_user?(activity, nil) || Enum.any?(x, &(&1 in y))
   end
 
diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
index 60738301b..cf7458d5f 100644
--- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
+++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
@@ -1518,9 +1518,9 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
     end
   end
 
-  def status_card(conn, %{"id" => status_id}) do
+  def status_card(%{assigns: %{user: user}} = conn, %{"id" => status_id}) do
     with %Activity{} = activity <- Repo.get(Activity, status_id),
-         true <- ActivityPub.is_public?(activity) do
+         true <- ActivityPub.visible_for_user?(activity, user) do
       data =
         StatusView.render(
           "card.json",
diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs
index 3dfbc8669..b52c2b805 100644
--- a/test/web/mastodon_api/mastodon_api_controller_test.exs
+++ b/test/web/mastodon_api/mastodon_api_controller_test.exs
@@ -1744,6 +1744,18 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do
                }
              }
 
+      # works with private posts
+      {:ok, activity} =
+        CommonAPI.post(user, %{"status" => "http://example.com/ogp", "visibility" => "direct"})
+
+      response_two =
+        conn
+        |> assign(:user, user)
+        |> get("/api/v1/statuses/#{activity.id}/card")
+        |> json_response(200)
+
+      assert response_two == response
+
       Pleroma.Config.put([:rich_media, :enabled], false)
     end
   end

From 9e0686efa61940fe78736c3e2669acb7233d8ada Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 22 Feb 2019 13:29:52 +0100
Subject: [PATCH 05/68] Move visibility into own module.

---
 lib/pleroma/gopher/server.ex                  |  3 +-
 lib/pleroma/web/activity_pub/activity_pub.ex  | 52 +---------
 .../activity_pub/activity_pub_controller.ex   |  9 +-
 .../web/activity_pub/transmogrifier.ex        |  3 +-
 lib/pleroma/web/activity_pub/visibility.ex    | 56 +++++++++++
 lib/pleroma/web/federator/federator.ex        |  3 +-
 .../mastodon_api/mastodon_api_controller.ex   | 11 ++-
 lib/pleroma/web/ostatus/ostatus_controller.ex |  9 +-
 lib/pleroma/web/streamer.ex                   |  6 +-
 .../web/twitter_api/twitter_api_controller.ex |  3 +-
 test/web/activity_pub/visibilty_test.exs      | 98 +++++++++++++++++++
 11 files changed, 182 insertions(+), 71 deletions(-)
 create mode 100644 lib/pleroma/web/activity_pub/visibility.ex
 create mode 100644 test/web/activity_pub/visibilty_test.exs

diff --git a/lib/pleroma/gopher/server.ex b/lib/pleroma/gopher/server.ex
index 32cb817d2..ba9614029 100644
--- a/lib/pleroma/gopher/server.ex
+++ b/lib/pleroma/gopher/server.ex
@@ -37,6 +37,7 @@ end
 
 defmodule Pleroma.Gopher.Server.ProtocolHandler do
   alias Pleroma.Web.ActivityPub.ActivityPub
+  alias Pleroma.Web.ActivityPub.Visibility
   alias Pleroma.Activity
   alias Pleroma.HTML
   alias Pleroma.User
@@ -110,7 +111,7 @@ defmodule Pleroma.Gopher.Server.ProtocolHandler do
 
   def response("/notices/" <> id) do
     with %Activity{} = activity <- Repo.get(Activity, id),
-         true <- ActivityPub.is_public?(activity) do
+         true <- Visibility.is_public?(activity) do
       activities =
         ActivityPub.fetch_activities_for_context(activity.data["context"])
         |> render_activities
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index fb0b7b68e..cc255cc9e 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -18,6 +18,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   import Ecto.Query
   import Pleroma.Web.ActivityPub.Utils
+  import Pleroma.Web.ActivityPub.Visibility
 
   require Logger
 
@@ -912,57 +913,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     end
   end
 
-  def is_public?(%Object{data: %{"type" => "Tombstone"}}), do: false
-  def is_public?(%Object{data: data}), do: is_public?(data)
-  def is_public?(%Activity{data: data}), do: is_public?(data)
-  def is_public?(%{"directMessage" => true}), do: false
-
-  def is_public?(data) do
-    "https://www.w3.org/ns/activitystreams#Public" in (data["to"] ++ (data["cc"] || []))
-  end
-
-  def is_private?(activity) do
-    unless is_public?(activity) do
-      follower_address = User.get_cached_by_ap_id(activity.data["actor"]).follower_address
-      Enum.any?(activity.data["to"], &(&1 == follower_address))
-    else
-      false
-    end
-  end
-
-  def is_direct?(%Activity{data: %{"directMessage" => true}}), do: true
-  def is_direct?(%Object{data: %{"directMessage" => true}}), do: true
-
-  def is_direct?(activity) do
-    !is_public?(activity) && !is_private?(activity)
-  end
-
-  def visible_for_user?(activity, nil) do
-    is_public?(activity)
-  end
-
-  def visible_for_user?(activity, user) do
-    x = [user.ap_id | user.following]
-    y = [activity.actor] ++ activity.data["to"] ++ (activity.data["cc"] || [])
-    visible_for_user?(activity, nil) || Enum.any?(x, &(&1 in y))
-  end
-
-  # guard
-  def entire_thread_visible_for_user?(nil, _user), do: false
-
-  # child
-  def entire_thread_visible_for_user?(
-        %Activity{data: %{"object" => %{"inReplyTo" => parent_id}}} = tail,
-        user
-      )
-      when is_binary(parent_id) do
-    parent = Activity.get_in_reply_to_activity(tail)
-    visible_for_user?(tail, user) && entire_thread_visible_for_user?(parent, user)
-  end
-
-  # root
-  def entire_thread_visible_for_user?(tail, user), do: visible_for_user?(tail, user)
-
   # filter out broken threads
   def contain_broken_threads(%Activity{} = activity, %User{} = user) do
     entire_thread_visible_for_user?(activity, user)
diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
index 2bea51311..ff924a536 100644
--- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
@@ -11,6 +11,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
   alias Pleroma.Web.ActivityPub.ObjectView
   alias Pleroma.Web.ActivityPub.UserView
   alias Pleroma.Web.ActivityPub.ActivityPub
+  alias Pleroma.Web.ActivityPub.Visibility
   alias Pleroma.Web.ActivityPub.Relay
   alias Pleroma.Web.ActivityPub.Transmogrifier
   alias Pleroma.Web.ActivityPub.Utils
@@ -49,7 +50,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
   def object(conn, %{"uuid" => uuid}) do
     with ap_id <- o_status_url(conn, :object, uuid),
          %Object{} = object <- Object.get_cached_by_ap_id(ap_id),
-         {_, true} <- {:public?, ActivityPub.is_public?(object)} do
+         {_, true} <- {:public?, Visibility.is_public?(object)} do
       conn
       |> put_resp_header("content-type", "application/activity+json")
       |> json(ObjectView.render("object.json", %{object: object}))
@@ -62,7 +63,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
   def object_likes(conn, %{"uuid" => uuid, "page" => page}) do
     with ap_id <- o_status_url(conn, :object, uuid),
          %Object{} = object <- Object.get_cached_by_ap_id(ap_id),
-         {_, true} <- {:public?, ActivityPub.is_public?(object)},
+         {_, true} <- {:public?, Visibility.is_public?(object)},
          likes <- Utils.get_object_likes(object) do
       {page, _} = Integer.parse(page)
 
@@ -78,7 +79,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
   def object_likes(conn, %{"uuid" => uuid}) do
     with ap_id <- o_status_url(conn, :object, uuid),
          %Object{} = object <- Object.get_cached_by_ap_id(ap_id),
-         {_, true} <- {:public?, ActivityPub.is_public?(object)},
+         {_, true} <- {:public?, Visibility.is_public?(object)},
          likes <- Utils.get_object_likes(object) do
       conn
       |> put_resp_header("content-type", "application/activity+json")
@@ -92,7 +93,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
   def activity(conn, %{"uuid" => uuid}) do
     with ap_id <- o_status_url(conn, :activity, uuid),
          %Activity{} = activity <- Activity.normalize(ap_id),
-         {_, true} <- {:public?, ActivityPub.is_public?(activity)} do
+         {_, true} <- {:public?, Visibility.is_public?(activity)} do
       conn
       |> put_resp_header("content-type", "application/activity+json")
       |> json(ObjectView.render("object.json", %{object: activity}))
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 41d89a02b..88007aa16 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -12,6 +12,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   alias Pleroma.Repo
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Utils
+  alias Pleroma.Web.ActivityPub.Visibility
 
   import Ecto.Query
 
@@ -489,7 +490,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     with actor <- get_actor(data),
          %User{} = actor <- User.get_or_fetch_by_ap_id(actor),
          {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id),
-         public <- ActivityPub.is_public?(data),
+         public <- Visibility.is_public?(data),
          {:ok, activity, _object} <- ActivityPub.announce(actor, object, id, false, public) do
       {:ok, activity}
     else
diff --git a/lib/pleroma/web/activity_pub/visibility.ex b/lib/pleroma/web/activity_pub/visibility.ex
new file mode 100644
index 000000000..db52fe933
--- /dev/null
+++ b/lib/pleroma/web/activity_pub/visibility.ex
@@ -0,0 +1,56 @@
+defmodule Pleroma.Web.ActivityPub.Visibility do
+  alias Pleroma.Activity
+  alias Pleroma.Object
+  alias Pleroma.User
+
+  def is_public?(%Object{data: %{"type" => "Tombstone"}}), do: false
+  def is_public?(%Object{data: data}), do: is_public?(data)
+  def is_public?(%Activity{data: data}), do: is_public?(data)
+  def is_public?(%{"directMessage" => true}), do: false
+
+  def is_public?(data) do
+    "https://www.w3.org/ns/activitystreams#Public" in (data["to"] ++ (data["cc"] || []))
+  end
+
+  def is_private?(activity) do
+    unless is_public?(activity) do
+      follower_address = User.get_cached_by_ap_id(activity.data["actor"]).follower_address
+      Enum.any?(activity.data["to"], &(&1 == follower_address))
+    else
+      false
+    end
+  end
+
+  def is_direct?(%Activity{data: %{"directMessage" => true}}), do: true
+  def is_direct?(%Object{data: %{"directMessage" => true}}), do: true
+
+  def is_direct?(activity) do
+    !is_public?(activity) && !is_private?(activity)
+  end
+
+  def visible_for_user?(activity, nil) do
+    is_public?(activity)
+  end
+
+  def visible_for_user?(activity, user) do
+    x = [user.ap_id | user.following]
+    y = [activity.actor] ++ activity.data["to"] ++ (activity.data["cc"] || [])
+    visible_for_user?(activity, nil) || Enum.any?(x, &(&1 in y))
+  end
+
+  # guard
+  def entire_thread_visible_for_user?(nil, _user), do: false
+
+  # child
+  def entire_thread_visible_for_user?(
+        %Activity{data: %{"object" => %{"inReplyTo" => parent_id}}} = tail,
+        user
+      )
+      when is_binary(parent_id) do
+    parent = Activity.get_in_reply_to_activity(tail)
+    visible_for_user?(tail, user) && entire_thread_visible_for_user?(parent, user)
+  end
+
+  # root
+  def entire_thread_visible_for_user?(tail, user), do: visible_for_user?(tail, user)
+end
diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex
index d4e2a9742..fbfe97dbc 100644
--- a/lib/pleroma/web/federator/federator.ex
+++ b/lib/pleroma/web/federator/federator.ex
@@ -9,6 +9,7 @@ defmodule Pleroma.Web.Federator do
   alias Pleroma.Web.Websub
   alias Pleroma.Web.Salmon
   alias Pleroma.Web.ActivityPub.ActivityPub
+  alias Pleroma.Web.ActivityPub.Visibility
   alias Pleroma.Web.ActivityPub.Relay
   alias Pleroma.Web.ActivityPub.Transmogrifier
   alias Pleroma.Web.ActivityPub.Utils
@@ -94,7 +95,7 @@ defmodule Pleroma.Web.Federator do
     with actor when not is_nil(actor) <- User.get_cached_by_ap_id(activity.data["actor"]) do
       {:ok, actor} = WebFinger.ensure_keys_present(actor)
 
-      if ActivityPub.is_public?(activity) do
+      if Visibility.is_public?(activity) do
         if OStatus.is_representable?(activity) do
           Logger.info(fn -> "Sending #{activity.data["id"]} out via WebSub" end)
           Websub.publish(Pleroma.Web.OStatus.feed_path(actor), actor, activity)
diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
index cf7458d5f..12987442a 100644
--- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
+++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
@@ -27,6 +27,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   alias Pleroma.Web.MastodonAPI.ReportView
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Utils
+  alias Pleroma.Web.ActivityPub.Visibility
   alias Pleroma.Web.OAuth.App
   alias Pleroma.Web.OAuth.Authorization
   alias Pleroma.Web.OAuth.Token
@@ -307,7 +308,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
 
   def get_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do
     with %Activity{} = activity <- Repo.get(Activity, id),
-         true <- ActivityPub.visible_for_user?(activity, user) do
+         true <- Visibility.visible_for_user?(activity, user) do
       conn
       |> put_view(StatusView)
       |> try_render("status.json", %{activity: activity, for: user})
@@ -449,7 +450,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   def bookmark_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do
     with %Activity{} = activity <- Repo.get(Activity, id),
          %User{} = user <- User.get_by_nickname(user.nickname),
-         true <- ActivityPub.visible_for_user?(activity, user),
+         true <- Visibility.visible_for_user?(activity, user),
          {:ok, user} <- User.bookmark(user, activity.data["object"]["id"]) do
       conn
       |> put_view(StatusView)
@@ -460,7 +461,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   def unbookmark_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do
     with %Activity{} = activity <- Repo.get(Activity, id),
          %User{} = user <- User.get_by_nickname(user.nickname),
-         true <- ActivityPub.visible_for_user?(activity, user),
+         true <- Visibility.visible_for_user?(activity, user),
          {:ok, user} <- User.unbookmark(user, activity.data["object"]["id"]) do
       conn
       |> put_view(StatusView)
@@ -867,7 +868,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
       if Regex.match?(~r/https?:/, query) do
         with {:ok, object} <- ActivityPub.fetch_object_from_id(query),
              %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]),
-             true <- ActivityPub.visible_for_user?(activity, user) do
+             true <- Visibility.visible_for_user?(activity, user) do
           [activity]
         else
           _e -> []
@@ -1520,7 +1521,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
 
   def status_card(%{assigns: %{user: user}} = conn, %{"id" => status_id}) do
     with %Activity{} = activity <- Repo.get(Activity, status_id),
-         true <- ActivityPub.visible_for_user?(activity, user) do
+         true <- Visibility.visible_for_user?(activity, user) do
       data =
         StatusView.render(
           "card.json",
diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex
index df723f638..4e963774a 100644
--- a/lib/pleroma/web/ostatus/ostatus_controller.ex
+++ b/lib/pleroma/web/ostatus/ostatus_controller.ex
@@ -9,6 +9,7 @@ defmodule Pleroma.Web.OStatus.OStatusController do
   alias Pleroma.Object
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
+  alias Pleroma.Web.ActivityPub.Visibility
   alias Pleroma.Web.ActivityPub.ActivityPubController
   alias Pleroma.Web.ActivityPub.ObjectView
   alias Pleroma.Web.OStatus.ActivityRepresenter
@@ -102,7 +103,7 @@ defmodule Pleroma.Web.OStatus.OStatusController do
     else
       with id <- o_status_url(conn, :object, uuid),
            {_, %Activity{} = activity} <- {:activity, Activity.get_create_by_object_ap_id(id)},
-           {_, true} <- {:public?, ActivityPub.is_public?(activity)},
+           {_, true} <- {:public?, Visibility.is_public?(activity)},
            %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do
         case get_format(conn) do
           "html" -> redirect(conn, to: "/notice/#{activity.id}")
@@ -127,7 +128,7 @@ defmodule Pleroma.Web.OStatus.OStatusController do
     else
       with id <- o_status_url(conn, :activity, uuid),
            {_, %Activity{} = activity} <- {:activity, Activity.normalize(id)},
-           {_, true} <- {:public?, ActivityPub.is_public?(activity)},
+           {_, true} <- {:public?, Visibility.is_public?(activity)},
            %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do
         case format = get_format(conn) do
           "html" -> redirect(conn, to: "/notice/#{activity.id}")
@@ -148,7 +149,7 @@ defmodule Pleroma.Web.OStatus.OStatusController do
 
   def notice(conn, %{"id" => id}) do
     with {_, %Activity{} = activity} <- {:activity, Activity.get_by_id(id)},
-         {_, true} <- {:public?, ActivityPub.is_public?(activity)},
+         {_, true} <- {:public?, Visibility.is_public?(activity)},
          %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do
       case format = get_format(conn) do
         "html" ->
@@ -191,7 +192,7 @@ defmodule Pleroma.Web.OStatus.OStatusController do
   # Returns an HTML embedded <audio> or <video> player suitable for embed iframes.
   def notice_player(conn, %{"id" => id}) do
     with %Activity{data: %{"type" => "Create"}} = activity <- Activity.get_by_id(id),
-         true <- ActivityPub.is_public?(activity),
+         true <- Visibility.is_public?(activity),
          %Object{} = object <- Object.normalize(activity.data["object"]),
          %{data: %{"attachment" => [%{"url" => [url | _]} | _]}} <- object,
          true <- String.starts_with?(url["mediaType"], ["audio", "video"]) do
diff --git a/lib/pleroma/web/streamer.ex b/lib/pleroma/web/streamer.ex
index 4de7608e4..477481bb9 100644
--- a/lib/pleroma/web/streamer.ex
+++ b/lib/pleroma/web/streamer.ex
@@ -10,7 +10,7 @@ defmodule Pleroma.Web.Streamer do
   alias Pleroma.Activity
   alias Pleroma.Object
   alias Pleroma.Repo
-  alias Pleroma.Web.ActivityPub.ActivityPub
+  alias Pleroma.Web.ActivityPub.Visibility
 
   @keepalive_interval :timer.seconds(30)
 
@@ -73,7 +73,7 @@ defmodule Pleroma.Web.Streamer do
   def handle_cast(%{action: :stream, topic: "list", item: item}, topics) do
     # filter the recipient list if the activity is not public, see #270.
     recipient_lists =
-      case ActivityPub.is_public?(item) do
+      case Visibility.is_public?(item) do
         true ->
           Pleroma.List.get_lists_from_activity(item)
 
@@ -82,7 +82,7 @@ defmodule Pleroma.Web.Streamer do
           |> Enum.filter(fn list ->
             owner = Repo.get(User, list.user_id)
 
-            ActivityPub.visible_for_user?(item, owner)
+            Visibility.visible_for_user?(item, owner)
           end)
       end
 
diff --git a/lib/pleroma/web/twitter_api/twitter_api_controller.ex b/lib/pleroma/web/twitter_api/twitter_api_controller.ex
index b815379fd..0d74c30c3 100644
--- a/lib/pleroma/web/twitter_api/twitter_api_controller.ex
+++ b/lib/pleroma/web/twitter_api/twitter_api_controller.ex
@@ -13,6 +13,7 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
   alias Pleroma.{Repo, Activity, Object, User, Notification}
   alias Pleroma.Web.OAuth.Token
   alias Pleroma.Web.ActivityPub.ActivityPub
+  alias Pleroma.Web.ActivityPub.Visibility
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.TwitterAPI.ActivityView
@@ -268,7 +269,7 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
 
   def fetch_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do
     with %Activity{} = activity <- Repo.get(Activity, id),
-         true <- ActivityPub.visible_for_user?(activity, user) do
+         true <- Visibility.visible_for_user?(activity, user) do
       conn
       |> put_view(ActivityView)
       |> render("activity.json", %{activity: activity, for: user})
diff --git a/test/web/activity_pub/visibilty_test.exs b/test/web/activity_pub/visibilty_test.exs
new file mode 100644
index 000000000..1172b7455
--- /dev/null
+++ b/test/web/activity_pub/visibilty_test.exs
@@ -0,0 +1,98 @@
+defmodule Pleroma.Web.ActivityPub.VisibilityTest do
+  use Pleroma.DataCase
+
+  alias Pleroma.Web.CommonAPI
+  alias Pleroma.Web.ActivityPub.Visibility
+  import Pleroma.Factory
+
+  setup do
+    user = insert(:user)
+    mentioned = insert(:user)
+    following = insert(:user)
+    unrelated = insert(:user)
+    {:ok, following} = Pleroma.User.follow(following, user)
+
+    {:ok, public} =
+      CommonAPI.post(user, %{"status" => "@#{mentioned.nickname}", "visibility" => "public"})
+
+    {:ok, private} =
+      CommonAPI.post(user, %{"status" => "@#{mentioned.nickname}", "visibility" => "private"})
+
+    {:ok, direct} =
+      CommonAPI.post(user, %{"status" => "@#{mentioned.nickname}", "visibility" => "direct"})
+
+    {:ok, unlisted} =
+      CommonAPI.post(user, %{"status" => "@#{mentioned.nickname}", "visibility" => "unlisted"})
+
+    %{
+      public: public,
+      private: private,
+      direct: direct,
+      unlisted: unlisted,
+      user: user,
+      mentioned: mentioned,
+      following: following,
+      unrelated: unrelated
+    }
+  end
+
+  test "is_direct?", %{public: public, private: private, direct: direct, unlisted: unlisted} do
+    assert Visibility.is_direct?(direct)
+    refute Visibility.is_direct?(public)
+    refute Visibility.is_direct?(private)
+    refute Visibility.is_direct?(unlisted)
+  end
+
+  test "is_public?", %{public: public, private: private, direct: direct, unlisted: unlisted} do
+    refute Visibility.is_public?(direct)
+    assert Visibility.is_public?(public)
+    refute Visibility.is_public?(private)
+    assert Visibility.is_public?(unlisted)
+  end
+
+  test "is_private?", %{public: public, private: private, direct: direct, unlisted: unlisted} do
+    refute Visibility.is_private?(direct)
+    refute Visibility.is_private?(public)
+    assert Visibility.is_private?(private)
+    refute Visibility.is_private?(unlisted)
+  end
+
+  test "visible_for_user?", %{
+    public: public,
+    private: private,
+    direct: direct,
+    unlisted: unlisted,
+    user: user,
+    mentioned: mentioned,
+    following: following,
+    unrelated: unrelated
+  } do
+    # All visible to author
+
+    assert Visibility.visible_for_user?(public, user)
+    assert Visibility.visible_for_user?(private, user)
+    assert Visibility.visible_for_user?(unlisted, user)
+    assert Visibility.visible_for_user?(direct, user)
+
+    # All visible to a mentioned user
+
+    assert Visibility.visible_for_user?(public, mentioned)
+    assert Visibility.visible_for_user?(private, mentioned)
+    assert Visibility.visible_for_user?(unlisted, mentioned)
+    assert Visibility.visible_for_user?(direct, mentioned)
+
+    # DM not visible for just follower
+
+    assert Visibility.visible_for_user?(public, following)
+    assert Visibility.visible_for_user?(private, following)
+    assert Visibility.visible_for_user?(unlisted, following)
+    refute Visibility.visible_for_user?(direct, following)
+
+    # Public and unlisted visible for unrelated user
+
+    assert Visibility.visible_for_user?(public, unrelated)
+    assert Visibility.visible_for_user?(unlisted, unrelated)
+    refute Visibility.visible_for_user?(private, unrelated)
+    refute Visibility.visible_for_user?(direct, unrelated)
+  end
+end

From bf562a6eccace1671cbba43d7489b66b1c48eae6 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Fri, 22 Feb 2019 17:26:47 +0000
Subject: [PATCH 06/68] Upate meck to fix tests on MacOS

---
 mix.lock | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mix.lock b/mix.lock
index e882f313e..5ffaedd16 100644
--- a/mix.lock
+++ b/mix.lock
@@ -34,7 +34,7 @@
   "jose": {:hex, :jose, "1.8.4", "7946d1e5c03a76ac9ef42a6e6a20001d35987afd68c2107bcd8f01a84e75aa73", [:mix, :rebar3], [{:base64url, "~> 0.0.1", [hex: :base64url, repo: "hexpm", optional: false]}], "hexpm"},
   "makeup": {:hex, :makeup, "0.5.5", "9e08dfc45280c5684d771ad58159f718a7b5788596099bdfb0284597d368a882", [:mix], [{:nimble_parsec, "~> 0.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm"},
   "makeup_elixir": {:hex, :makeup_elixir, "0.10.0", "0f09c2ddf352887a956d84f8f7e702111122ca32fbbc84c2f0569b8b65cbf7fa", [:mix], [{:makeup, "~> 0.5.5", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm"},
-  "meck": {:hex, :meck, "0.8.9", "64c5c0bd8bcca3a180b44196265c8ed7594e16bcc845d0698ec6b4e577f48188", [:rebar3], [], "hexpm"},
+  "meck": {:hex, :meck, "0.8.13", "ffedb39f99b0b99703b8601c6f17c7f76313ee12de6b646e671e3188401f7866", [:rebar3], [], "hexpm"},
   "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm"},
   "mime": {:hex, :mime, "1.3.1", "30ce04ab3175b6ad0bdce0035cba77bba68b813d523d1aac73d9781b4d193cf8", [:mix], [], "hexpm"},
   "mimerl": {:hex, :mimerl, "1.0.2", "993f9b0e084083405ed8252b99460c4f0563e41729ab42d9074fd5e52439be88", [:rebar3], [], "hexpm"},

From c75e643a72a3b27340877f21d7d9fe31dbfd146e Mon Sep 17 00:00:00 2001
From: faried nawaz <faried@gmail.com>
Date: Sat, 23 Feb 2019 00:18:03 +0500
Subject: [PATCH 07/68] apache config: add instructions for mediaproxy.

---
 installation/pleroma-apache.conf | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/installation/pleroma-apache.conf b/installation/pleroma-apache.conf
index d5e75044f..2beb7c4cc 100644
--- a/installation/pleroma-apache.conf
+++ b/installation/pleroma-apache.conf
@@ -1,6 +1,7 @@
 # default Apache site config for Pleroma
 #
 # needed modules: define headers proxy proxy_http proxy_wstunnel rewrite ssl
+# optional modules: cache cache_disk
 #
 # Simple installation instructions:
 # 1. Install your TLS certificate, possibly using Let's Encrypt.
@@ -8,6 +9,14 @@
 # 3. This assumes a Debian style Apache config. Copy this file to
 #    /etc/apache2/sites-available/ and then add a symlink to it in
 #    /etc/apache2/sites-enabled/ by running 'a2ensite pleroma-apache.conf', then restart Apache.
+#
+# Optional: enable disk-based caching for the media proxy
+# For details, see https://git.pleroma.social/pleroma/pleroma/wikis/How%20to%20activate%20mediaproxy
+#
+# 1. Create the directory listed below as the CacheRoot, and make sure
+#    the Apache user can write to it.
+# 2. Configure Apache's htcacheclean to clean the directory periodically.
+# 3. Run 'a2enmod cache cache_disk' and restart Apache.
 
 Define servername example.tld
 
@@ -34,6 +43,15 @@ CustomLog ${APACHE_LOG_DIR}/access.log combined
     SSLCompression          off
     SSLSessionTickets       off
 
+    # uncomment the following to enable mediaproxy caching on disk
+    # <IfModule mod_cache_disk.c>
+    #     CacheRoot /var/cache/apache2/mod_cache_disk
+    #     CacheDirLevels 1
+    #     CacheDirLength 2
+    #     CacheEnable disk /proxy
+    #     CacheLock on
+    # </IfModule>
+
     RewriteEngine On
     RewriteCond %{HTTP:Connection} Upgrade [NC]
     RewriteCond %{HTTP:Upgrade} websocket [NC]

From efa506b874a0caf85800ec91f1aef4b2878840d3 Mon Sep 17 00:00:00 2001
From: 11backslashes <reversesolidi@protonmail.com>
Date: Sun, 24 Feb 2019 13:23:47 -0500
Subject: [PATCH 08/68] delegate user retrieval to the User module

---
 lib/pleroma/web/twitter_api/twitter_api.ex | 10 +---------
 1 file changed, 1 insertion(+), 9 deletions(-)

diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex
index efdd0bf43..ab6470d78 100644
--- a/lib/pleroma/web/twitter_api/twitter_api.ex
+++ b/lib/pleroma/web/twitter_api/twitter_api.ex
@@ -229,18 +229,10 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do
     end
   end
 
-  def get_by_id_or_nickname(id_or_nickname) do
-    if !is_integer(id_or_nickname) && :error == Integer.parse(id_or_nickname) do
-      Repo.get_by(User, nickname: id_or_nickname)
-    else
-      Repo.get(User, id_or_nickname)
-    end
-  end
-
   def get_user(user \\ nil, params) do
     case params do
       %{"user_id" => user_id} ->
-        case target = get_by_id_or_nickname(user_id) do
+        case target = User.get_cached_by_nickname_or_id(user_id) do
           nil ->
             {:error, "No user with such user_id"}
 

From fc9d895ebc71cb89192e23f7dca0a7fabaa2cdab Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Sun, 24 Feb 2019 18:53:49 +0000
Subject: [PATCH 09/68] twitter card: remove IO.inspect() spam

---
 lib/pleroma/web/metadata/twitter_card.ex | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/lib/pleroma/web/metadata/twitter_card.ex b/lib/pleroma/web/metadata/twitter_card.ex
index d672b397f..a0be383e5 100644
--- a/lib/pleroma/web/metadata/twitter_card.ex
+++ b/lib/pleroma/web/metadata/twitter_card.ex
@@ -66,9 +66,7 @@ defmodule Pleroma.Web.Metadata.Providers.TwitterCard do
     end
   end
 
-  defp build_attachments(id, z = %{data: %{"attachment" => attachments}}) do
-    IO.puts(inspect(z))
-
+  defp build_attachments(id, %{data: %{"attachment" => attachments}}) do
     Enum.reduce(attachments, [], fn attachment, acc ->
       rendered_tags =
         Enum.reduce(attachment["url"], [], fn url, acc ->

From d3a6c065a496c3a7c4f0604af646b6fb611f585b Mon Sep 17 00:00:00 2001
From: PEA <phenethylamine@tutanota.com>
Date: Mon, 25 Feb 2019 23:37:46 -0500
Subject: [PATCH 10/68] Add ipv6 handling to pleroma.nginx

Replaces `listen 80` with `listen [::]:80`, same with 443
---
 installation/pleroma.nginx | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/installation/pleroma.nginx b/installation/pleroma.nginx
index a3d55e4bf..66a0e987c 100644
--- a/installation/pleroma.nginx
+++ b/installation/pleroma.nginx
@@ -11,7 +11,7 @@ proxy_cache_path /tmp/pleroma-media-cache levels=1:2 keys_zone=pleroma_media_cac
 
 server {
     server_name    example.tld;
-    listen         80;
+    listen         [::]:80;
     return         301 https://$server_name$request_uri;
 
     # Uncomment this if you need to use the 'webroot' method with certbot. Make sure
@@ -29,7 +29,7 @@ server {
 ssl_session_cache shared:ssl_session_cache:10m;
 
 server {
-    listen 443 ssl http2;
+    listen [::]:443 ssl http2;
     ssl_session_timeout 5m;
 
     ssl_trusted_certificate   /etc/letsencrypt/live/example.tld/fullchain.pem;

From e82b70eb530293c3dfe8597c4100320fba96e479 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Tue, 26 Feb 2019 15:27:01 +0300
Subject: [PATCH 11/68] Database authenticator behaviour / Pleroma
 implementation refactoring.

---
 lib/pleroma/web/auth/database_authenticator.ex     | 14 ++++++++++++++
 .../pleroma_database_authenticator.ex}             |  4 ++--
 lib/pleroma/web/oauth.ex                           |  8 --------
 lib/pleroma/web/oauth/authenticator_adapter.ex     |  7 -------
 lib/pleroma/web/oauth/oauth_controller.ex          |  6 +++---
 5 files changed, 19 insertions(+), 20 deletions(-)
 create mode 100644 lib/pleroma/web/auth/database_authenticator.ex
 rename lib/pleroma/web/{oauth/authenticator.ex => auth/pleroma_database_authenticator.ex} (81%)
 delete mode 100644 lib/pleroma/web/oauth/authenticator_adapter.ex

diff --git a/lib/pleroma/web/auth/database_authenticator.ex b/lib/pleroma/web/auth/database_authenticator.ex
new file mode 100644
index 000000000..69024a4ba
--- /dev/null
+++ b/lib/pleroma/web/auth/database_authenticator.ex
@@ -0,0 +1,14 @@
+defmodule Pleroma.Web.Auth.DatabaseAuthenticator do
+  alias Pleroma.User
+
+  @implementation Pleroma.Config.get(
+                    Pleroma.Web.Auth.DatabaseAuthenticator,
+                    Pleroma.Web.Auth.PleromaDatabaseAuthenticator
+                  )
+
+  @callback get_user(Plug.Conn.t()) :: {:ok, User.t()} | {:error, any()}
+  defdelegate get_user(plug), to: @implementation
+
+  @callback handle_error(Plug.Conn.t(), any()) :: any()
+  defdelegate handle_error(plug, error), to: @implementation
+end
diff --git a/lib/pleroma/web/oauth/authenticator.ex b/lib/pleroma/web/auth/pleroma_database_authenticator.ex
similarity index 81%
rename from lib/pleroma/web/oauth/authenticator.ex
rename to lib/pleroma/web/auth/pleroma_database_authenticator.ex
index 86bbc41f0..79a8dcfce 100644
--- a/lib/pleroma/web/oauth/authenticator.ex
+++ b/lib/pleroma/web/auth/pleroma_database_authenticator.ex
@@ -1,8 +1,8 @@
-defmodule Pleroma.Web.Authenticator do
+defmodule Pleroma.Web.Auth.PleromaDatabaseAuthenticator do
   alias Pleroma.User
   alias Comeonin.Pbkdf2
 
-  @behaviour Pleroma.Web.AuthenticatorAdapter
+  @behaviour Pleroma.Web.Auth.DatabaseAuthenticator
 
   def get_user(%Plug.Conn{} = conn) do
     %{"authorization" => %{"name" => name, "password" => password}} = conn.params
diff --git a/lib/pleroma/web/oauth.ex b/lib/pleroma/web/oauth.ex
index f3bac33c8..d2835a0ba 100644
--- a/lib/pleroma/web/oauth.ex
+++ b/lib/pleroma/web/oauth.ex
@@ -3,14 +3,6 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.OAuth do
-  @authenticator Application.get_env(
-                   :pleroma,
-                   Pleroma.Web.AuthenticatorAdapter,
-                   Pleroma.Web.Authenticator
-                 )
-
-  def authenticator, do: @authenticator
-
   def parse_scopes(scopes, _default) when is_list(scopes) do
     Enum.filter(scopes, &(&1 not in [nil, ""]))
   end
diff --git a/lib/pleroma/web/oauth/authenticator_adapter.ex b/lib/pleroma/web/oauth/authenticator_adapter.ex
deleted file mode 100644
index 282963b1c..000000000
--- a/lib/pleroma/web/oauth/authenticator_adapter.ex
+++ /dev/null
@@ -1,7 +0,0 @@
-defmodule Pleroma.Web.AuthenticatorAdapter do
-  alias Pleroma.User
-
-  @callback get_user(Plug.Conn.t()) :: {:ok, User.t()} | {:error, any()}
-
-  @callback handle_error(Plug.Conn.t(), any()) :: any()
-end
diff --git a/lib/pleroma/web/oauth/oauth_controller.ex b/lib/pleroma/web/oauth/oauth_controller.ex
index abe6fd2f2..02c0babd2 100644
--- a/lib/pleroma/web/oauth/oauth_controller.ex
+++ b/lib/pleroma/web/oauth/oauth_controller.ex
@@ -5,7 +5,7 @@
 defmodule Pleroma.Web.OAuth.OAuthController do
   use Pleroma.Web, :controller
 
-  alias Pleroma.Web.OAuth
+  alias Pleroma.Web.Auth.DatabaseAuthenticator
   alias Pleroma.Web.OAuth.Authorization
   alias Pleroma.Web.OAuth.Token
   alias Pleroma.Web.OAuth.App
@@ -45,7 +45,7 @@ defmodule Pleroma.Web.OAuth.OAuthController do
             "redirect_uri" => redirect_uri
           } = auth_params
       }) do
-    with {_, {:ok, %User{} = user}} <- {:get_user, OAuth.authenticator().get_user(conn)},
+    with {_, {:ok, %User{} = user}} <- {:get_user, DatabaseAuthenticator.get_user(conn)},
          %App{} = app <- Repo.get_by(App, client_id: client_id),
          true <- redirect_uri in String.split(app.redirect_uris),
          scopes <- oauth_scopes(auth_params, []),
@@ -98,7 +98,7 @@ defmodule Pleroma.Web.OAuth.OAuthController do
         |> authorize(auth_params)
 
       error ->
-        OAuth.authenticator().handle_error(conn, error)
+        DatabaseAuthenticator.handle_error(conn, error)
     end
   end
 

From e98d34e5fb569b52166f74a1fc255310938d657c Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Tue, 26 Feb 2019 16:26:54 +0300
Subject: [PATCH 12/68] Added missing copyright headers.

---
 lib/pleroma/web/auth/database_authenticator.ex         | 4 ++++
 lib/pleroma/web/auth/pleroma_database_authenticator.ex | 4 ++++
 2 files changed, 8 insertions(+)

diff --git a/lib/pleroma/web/auth/database_authenticator.ex b/lib/pleroma/web/auth/database_authenticator.ex
index 69024a4ba..02a16b634 100644
--- a/lib/pleroma/web/auth/database_authenticator.ex
+++ b/lib/pleroma/web/auth/database_authenticator.ex
@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Web.Auth.DatabaseAuthenticator do
   alias Pleroma.User
 
diff --git a/lib/pleroma/web/auth/pleroma_database_authenticator.ex b/lib/pleroma/web/auth/pleroma_database_authenticator.ex
index 79a8dcfce..39aa1a586 100644
--- a/lib/pleroma/web/auth/pleroma_database_authenticator.ex
+++ b/lib/pleroma/web/auth/pleroma_database_authenticator.ex
@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Web.Auth.PleromaDatabaseAuthenticator do
   alias Pleroma.User
   alias Comeonin.Pbkdf2

From e1bdaaa3fe20ad10458afa5edfb161fedcea74c0 Mon Sep 17 00:00:00 2001
From: Kenneth Zhao <ken@epenguin.com>
Date: Tue, 26 Feb 2019 08:41:37 -0800
Subject: [PATCH 13/68] need to put back ipv4 listen instruct

---
 installation/pleroma.nginx | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/installation/pleroma.nginx b/installation/pleroma.nginx
index 66a0e987c..8709f2cb7 100644
--- a/installation/pleroma.nginx
+++ b/installation/pleroma.nginx
@@ -11,6 +11,8 @@ proxy_cache_path /tmp/pleroma-media-cache levels=1:2 keys_zone=pleroma_media_cac
 
 server {
     server_name    example.tld;
+
+    listen         80;
     listen         [::]:80;
     return         301 https://$server_name$request_uri;
 
@@ -29,6 +31,9 @@ server {
 ssl_session_cache shared:ssl_session_cache:10m;
 
 server {
+    server_name example.tld;
+
+    listen 443 ssl http2;
     listen [::]:443 ssl http2;
     ssl_session_timeout 5m;
 
@@ -48,8 +53,6 @@ server {
     ssl_stapling on;
     ssl_stapling_verify on;
 
-    server_name example.tld;
-
     gzip_vary on;
     gzip_proxied any;
     gzip_comp_level 6;

From 90d0d055fccc630066e66627281432e382aefdb1 Mon Sep 17 00:00:00 2001
From: Maxim Filippov <colixer@gmail.com>
Date: Wed, 27 Feb 2019 00:13:38 +0300
Subject: [PATCH 14/68] Add more admin actions

---
 lib/pleroma/user.ex                             |  6 ++++++
 .../web/admin_api/admin_api_controller.ex       | 17 +++++++++++++++++
 lib/pleroma/web/router.ex                       |  2 ++
 lib/pleroma/web/twitter_api/views/user_view.ex  | 14 ++++++++++++++
 4 files changed, 39 insertions(+)

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index c98b942ff..12e0e818e 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -772,6 +772,12 @@ defmodule Pleroma.User do
     Enum.uniq_by(fts_results ++ trigram_results, & &1.id)
   end
 
+  def all_except_one(user) do
+    query = from(u in User, where: u.id != ^user.id)
+
+    Repo.all(query)
+  end
+
   defp do_search(subquery, for_user, options \\ []) do
     q =
       from(
diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex
index 9ec50bb90..ef72509fe 100644
--- a/lib/pleroma/web/admin_api/admin_api_controller.ex
+++ b/lib/pleroma/web/admin_api/admin_api_controller.ex
@@ -6,6 +6,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
   use Pleroma.Web, :controller
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.Relay
+  alias Pleroma.Web.TwitterAPI.UserView
 
   import Pleroma.Web.ControllerHelper, only: [json_response: 3]
 
@@ -41,6 +42,15 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     |> json(user.nickname)
   end
 
+  def user_toggle_activation(conn, %{"nickname" => nickname}) do
+    user = User.get_by_nickname(nickname)
+
+    {:ok, updated_user} = User.deactivate(user, !user.info.deactivated)
+
+    conn
+    |> json(UserView.render("show_for_admin.json", %{user: updated_user}))
+  end
+
   def tag_users(conn, %{"nicknames" => nicknames, "tags" => tags}) do
     with {:ok, _} <- User.tag(nicknames, tags),
          do: json_response(conn, :no_content, "")
@@ -51,6 +61,13 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
          do: json_response(conn, :no_content, "")
   end
 
+  def list_users(%{assigns: %{user: admin}} = conn, _data) do
+    users = User.all_except_one(admin)
+
+    conn
+    |> json(UserView.render("index_for_admin.json", %{users: users}))
+  end
+
   def right_add(conn, %{"permission_group" => permission_group, "nickname" => nickname})
       when permission_group in ["moderator", "admin"] do
     user = User.get_by_nickname(nickname)
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index 5aebcb353..3b1fd46a5 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -139,7 +139,9 @@ defmodule Pleroma.Web.Router do
   scope "/api/pleroma/admin", Pleroma.Web.AdminAPI do
     pipe_through([:admin_api, :oauth_write])
 
+    get("/users", AdminAPIController, :list_users)
     delete("/user", AdminAPIController, :user_delete)
+    patch("/users/:nickname/toggle_activation", AdminAPIController, :user_toggle_activation)
     post("/user", AdminAPIController, :user_create)
     put("/users/tag", AdminAPIController, :tag_users)
     delete("/users/tag", AdminAPIController, :untag_users)
diff --git a/lib/pleroma/web/twitter_api/views/user_view.ex b/lib/pleroma/web/twitter_api/views/user_view.ex
index df7384476..c5034cf36 100644
--- a/lib/pleroma/web/twitter_api/views/user_view.ex
+++ b/lib/pleroma/web/twitter_api/views/user_view.ex
@@ -9,6 +9,7 @@ defmodule Pleroma.Web.TwitterAPI.UserView do
   alias Pleroma.User
   alias Pleroma.Web.CommonAPI.Utils
   alias Pleroma.Web.MediaProxy
+  alias Pleroma.Web.TwitterAPI.UserView
 
   def render("show.json", %{user: user = %User{}} = assigns) do
     render_one(user, Pleroma.Web.TwitterAPI.UserView, "user.json", assigns)
@@ -26,6 +27,19 @@ defmodule Pleroma.Web.TwitterAPI.UserView do
       else: %{}
   end
 
+  def render("index_for_admin.json", %{users: users} = opts) do
+    users
+    |> render_many(UserView, "show_for_admin.json", opts)
+  end
+
+  def render("show_for_admin.json", %{user: user}) do
+    %{
+      "id" => user.id,
+      "nickname" => user.nickname,
+      "deactivated" => user.info.deactivated
+    }
+  end
+
   def render("short.json", %{
         user: %User{
           nickname: nickname,

From c3ac9424d2affe87df82c14dc243f507fa639343 Mon Sep 17 00:00:00 2001
From: Egor <egor@kislitsyn.com>
Date: Tue, 26 Feb 2019 23:32:26 +0000
Subject: [PATCH 15/68] AutoLinker

---
 config/config.exs                             |  10 +
 docs/config.md                                |  27 ++-
 lib/pleroma/formatter.ex                      | 194 +++++-------------
 lib/pleroma/user.ex                           |   6 +-
 lib/pleroma/web/common_api/common_api.ex      |  28 +--
 lib/pleroma/web/common_api/utils.ex           |  86 ++++----
 mix.exs                                       |   5 +-
 mix.lock                                      |   1 +
 test/formatter_test.exs                       | 112 ++++------
 test/web/common_api/common_api_utils_test.exs |  32 ++-
 10 files changed, 211 insertions(+), 290 deletions(-)

diff --git a/config/config.exs b/config/config.exs
index 6119aaea1..7e4ac1100 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -344,6 +344,16 @@ config :pleroma, Pleroma.Jobs,
   federator_outgoing: [max_jobs: 50],
   mailer: [max_jobs: 10]
 
+config :auto_linker,
+  opts: [
+    scheme: true,
+    extra: true,
+    class: false,
+    strip_prefix: false,
+    new_window: false,
+    rel: false
+  ]
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"
diff --git a/docs/config.md b/docs/config.md
index 14723b727..d1bf2a6f4 100644
--- a/docs/config.md
+++ b/docs/config.md
@@ -107,7 +107,7 @@ config :pleroma, Pleroma.Mailer,
 
 An example to enable ONLY ExSyslogger (f/ex in ``prod.secret.exs``) with info and debug suppressed:
 ```
-config :logger, 
+config :logger,
   backends: [{ExSyslogger, :ex_syslogger}]
 
 config :logger, :ex_syslogger,
@@ -301,3 +301,28 @@ For each pool, the options are:
 * `max_connections` - how much connections a pool can hold
 * `timeout` - retention duration for connections
 
+## :auto_linker
+
+Configuration for the `auto_linker` library:
+
+* `class: "auto-linker"` - specify the class to be added to the generated link. false to clear
+* `rel: "noopener noreferrer"` - override the rel attribute. false to clear
+* `new_window: true` - set to false to remove `target='_blank'` attribute
+* `scheme: false` - Set to true to link urls with schema `http://google.com`
+* `truncate: false` - Set to a number to truncate urls longer then the number. Truncated urls will end in `..`
+* `strip_prefix: true` - Strip the scheme prefix
+* `extra: false` - link urls with rarely used schemes (magnet, ipfs, irc, etc.)
+
+Example:
+
+```exs
+config :auto_linker,
+  opts: [
+    scheme: true,
+    extra: true,
+    class: false,
+    strip_prefix: false,
+    new_window: false,
+    rel: false
+  ]
+```
diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex
index f31aafa0d..51d08c5ee 100644
--- a/lib/pleroma/formatter.ex
+++ b/lib/pleroma/formatter.ex
@@ -8,33 +8,51 @@ defmodule Pleroma.Formatter do
   alias Pleroma.User
   alias Pleroma.Web.MediaProxy
 
-  @tag_regex ~r/((?<=[^&])|\A)(\#)(\w+)/u
   @markdown_characters_regex ~r/(`|\*|_|{|}|[|]|\(|\)|#|\+|-|\.|!)/
+  @link_regex ~r{((?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~%:/?#[\]@!\$&'\(\)\*\+,;=.]+)|[0-9a-z+\-\.]+:[0-9a-z$-_.+!*'(),]+}ui
 
-  # Modified from https://www.w3.org/TR/html5/forms.html#valid-e-mail-address
-  @mentions_regex ~r/@[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]*@?[a-zA-Z0-9_-](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*/u
+  @auto_linker_config hashtag: true,
+                      hashtag_handler: &Pleroma.Formatter.hashtag_handler/4,
+                      mention: true,
+                      mention_handler: &Pleroma.Formatter.mention_handler/4
 
-  def parse_tags(text, data \\ %{}) do
-    Regex.scan(@tag_regex, text)
-    |> Enum.map(fn ["#" <> tag = full_tag | _] -> {full_tag, String.downcase(tag)} end)
-    |> (fn map ->
-          if data["sensitive"] in [true, "True", "true", "1"],
-            do: [{"#nsfw", "nsfw"}] ++ map,
-            else: map
-        end).()
+  def mention_handler("@" <> nickname, buffer, opts, acc) do
+    case User.get_cached_by_nickname(nickname) do
+      %User{id: id} = user ->
+        ap_id = get_ap_id(user)
+        nickname_text = get_nickname_text(nickname, opts) |> maybe_escape(opts)
+
+        link =
+          "<span class='h-card'><a data-user='#{id}' class='u-url mention' href='#{ap_id}'>@<span>#{
+            nickname_text
+          }</span></a></span>"
+
+        {link, %{acc | mentions: MapSet.put(acc.mentions, {"@" <> nickname, user})}}
+
+      _ ->
+        {buffer, acc}
+    end
   end
 
-  @doc "Parses mentions text and returns list {nickname, user}."
-  @spec parse_mentions(binary()) :: list({binary(), User.t()})
-  def parse_mentions(text) do
-    Regex.scan(@mentions_regex, text)
-    |> List.flatten()
-    |> Enum.uniq()
-    |> Enum.map(fn nickname ->
-      with nickname <- String.trim_leading(nickname, "@"),
-           do: {"@" <> nickname, User.get_cached_by_nickname(nickname)}
-    end)
-    |> Enum.filter(fn {_match, user} -> user end)
+  def hashtag_handler("#" <> tag = tag_text, _buffer, _opts, acc) do
+    tag = String.downcase(tag)
+    url = "#{Pleroma.Web.base_url()}/tag/#{tag}"
+    link = "<a class='hashtag' data-tag='#{tag}' href='#{url}' rel='tag'>#{tag_text}</a>"
+
+    {link, %{acc | tags: MapSet.put(acc.tags, {tag_text, tag})}}
+  end
+
+  @doc """
+  Parses a text and replace plain text links with HTML. Returns a tuple with a result text, mentions, and hashtags.
+  """
+  @spec linkify(String.t(), keyword()) ::
+          {String.t(), [{String.t(), User.t()}], [{String.t(), String.t()}]}
+  def linkify(text, options \\ []) do
+    options = options ++ @auto_linker_config
+    acc = %{mentions: MapSet.new(), tags: MapSet.new()}
+    {text, %{mentions: mentions, tags: tags}} = AutoLinker.link_map(text, acc, options)
+
+    {text, MapSet.to_list(mentions), MapSet.to_list(tags)}
   end
 
   def emojify(text) do
@@ -48,9 +66,7 @@ defmodule Pleroma.Formatter do
       emoji = HTML.strip_tags(emoji)
       file = HTML.strip_tags(file)
 
-      String.replace(
-        text,
-        ":#{emoji}:",
+      html =
         if not strip do
           "<img height='32px' width='32px' alt='#{emoji}' title='#{emoji}' src='#{
             MediaProxy.url(file)
@@ -58,8 +74,8 @@ defmodule Pleroma.Formatter do
         else
           ""
         end
-      )
-      |> HTML.filter_tags()
+
+      String.replace(text, ":#{emoji}:", html) |> HTML.filter_tags()
     end)
   end
 
@@ -75,12 +91,6 @@ defmodule Pleroma.Formatter do
 
   def get_emoji(_), do: []
 
-  @link_regex ~r/[0-9a-z+\-\.]+:[0-9a-z$-_.+!*'(),]+/ui
-
-  @uri_schemes Application.get_env(:pleroma, :uri_schemes, [])
-  @valid_schemes Keyword.get(@uri_schemes, :valid_schemes, [])
-
-  # TODO: make it use something other than @link_regex
   def html_escape(text, "text/html") do
     HTML.filter_tags(text)
   end
@@ -94,112 +104,6 @@ defmodule Pleroma.Formatter do
     |> Enum.join("")
   end
 
-  @doc """
-  Escapes a special characters in mention names.
-  """
-  @spec mentions_escape(String.t(), list({String.t(), any()})) :: String.t()
-  def mentions_escape(text, mentions) do
-    mentions
-    |> Enum.reduce(text, fn {name, _}, acc ->
-      escape_name = String.replace(name, @markdown_characters_regex, "\\\\\\1")
-      String.replace(acc, name, escape_name)
-    end)
-  end
-
-  @doc "changes scheme:... urls to html links"
-  def add_links({subs, text}) do
-    links =
-      text
-      |> String.split([" ", "\t", "<br>"])
-      |> Enum.filter(fn word -> String.starts_with?(word, @valid_schemes) end)
-      |> Enum.filter(fn word -> Regex.match?(@link_regex, word) end)
-      |> Enum.map(fn url -> {Ecto.UUID.generate(), url} end)
-      |> Enum.sort_by(fn {_, url} -> -String.length(url) end)
-
-    uuid_text =
-      links
-      |> Enum.reduce(text, fn {uuid, url}, acc -> String.replace(acc, url, uuid) end)
-
-    subs =
-      subs ++
-        Enum.map(links, fn {uuid, url} ->
-          {uuid, "<a href=\"#{url}\">#{url}</a>"}
-        end)
-
-    {subs, uuid_text}
-  end
-
-  @doc "Adds the links to mentioned users"
-  def add_user_links({subs, text}, mentions, options \\ []) do
-    mentions =
-      mentions
-      |> Enum.sort_by(fn {name, _} -> -String.length(name) end)
-      |> Enum.map(fn {name, user} -> {name, user, Ecto.UUID.generate()} end)
-
-    uuid_text =
-      mentions
-      |> Enum.reduce(text, fn {match, _user, uuid}, text ->
-        String.replace(text, match, uuid)
-      end)
-
-    subs =
-      subs ++
-        Enum.map(mentions, fn {match, %User{id: id, ap_id: ap_id, info: info}, uuid} ->
-          ap_id =
-            if is_binary(info.source_data["url"]) do
-              info.source_data["url"]
-            else
-              ap_id
-            end
-
-          nickname =
-            if options[:format] == :full do
-              User.full_nickname(match)
-            else
-              User.local_nickname(match)
-            end
-
-          {uuid,
-           "<span class='h-card'><a data-user='#{id}' class='u-url mention' href='#{ap_id}'>" <>
-             "@<span>#{nickname}</span></a></span>"}
-        end)
-
-    {subs, uuid_text}
-  end
-
-  @doc "Adds the hashtag links"
-  def add_hashtag_links({subs, text}, tags) do
-    tags =
-      tags
-      |> Enum.sort_by(fn {name, _} -> -String.length(name) end)
-      |> Enum.map(fn {name, short} -> {name, short, Ecto.UUID.generate()} end)
-
-    uuid_text =
-      tags
-      |> Enum.reduce(text, fn {match, _short, uuid}, text ->
-        String.replace(text, ~r/((?<=[^&])|(\A))#{match}/, uuid)
-      end)
-
-    subs =
-      subs ++
-        Enum.map(tags, fn {tag_text, tag, uuid} ->
-          url =
-            "<a class='hashtag' data-tag='#{tag}' href='#{Pleroma.Web.base_url()}/tag/#{tag}' rel='tag'>#{
-              tag_text
-            }</a>"
-
-          {uuid, url}
-        end)
-
-    {subs, uuid_text}
-  end
-
-  def finalize({subs, text}) do
-    Enum.reduce(subs, text, fn {uuid, replacement}, result_text ->
-      String.replace(result_text, uuid, replacement)
-    end)
-  end
-
   def truncate(text, max_length \\ 200, omission \\ "...") do
     # Remove trailing whitespace
     text = Regex.replace(~r/([^ \t\r\n])([ \t]+$)/u, text, "\\g{1}")
@@ -211,4 +115,16 @@ defmodule Pleroma.Formatter do
       String.slice(text, 0, length_with_omission) <> omission
     end
   end
+
+  defp get_ap_id(%User{info: %{source_data: %{"url" => url}}}) when is_binary(url), do: url
+  defp get_ap_id(%User{ap_id: ap_id}), do: ap_id
+
+  defp get_nickname_text(nickname, %{mentions_format: :full}), do: User.full_nickname(nickname)
+  defp get_nickname_text(nickname, _), do: User.local_nickname(nickname)
+
+  defp maybe_escape(str, %{mentions_escape: true}) do
+    String.replace(str, @markdown_characters_regex, "\\\\\\1")
+  end
+
+  defp maybe_escape(str, _), do: str
 end
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 12e0e818e..01d532ab3 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -1193,9 +1193,6 @@ defmodule Pleroma.User do
   def parse_bio(bio, _user) when bio == "", do: bio
 
   def parse_bio(bio, user) do
-    mentions = Formatter.parse_mentions(bio)
-    tags = Formatter.parse_tags(bio)
-
     emoji =
       (user.info.source_data["tag"] || [])
       |> Enum.filter(fn %{"type" => t} -> t == "Emoji" end)
@@ -1204,7 +1201,8 @@ defmodule Pleroma.User do
       end)
 
     bio
-    |> CommonUtils.format_input(mentions, tags, "text/plain", user_links: [format: :full])
+    |> CommonUtils.format_input("text/plain", mentions_format: :full)
+    |> elem(0)
     |> Formatter.emojify(emoji)
   end
 
diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex
index e788337cc..7114d6de6 100644
--- a/lib/pleroma/web/common_api/common_api.ex
+++ b/lib/pleroma/web/common_api/common_api.ex
@@ -82,40 +82,20 @@ defmodule Pleroma.Web.CommonAPI do
 
   def get_visibility(_), do: "public"
 
-  defp get_content_type(content_type) do
-    if Enum.member?(Pleroma.Config.get([:instance, :allowed_post_formats]), content_type) do
-      content_type
-    else
-      "text/plain"
-    end
-  end
-
   def post(user, %{"status" => status} = data) do
     visibility = get_visibility(data)
     limit = Pleroma.Config.get([:instance, :limit])
 
     with status <- String.trim(status),
          attachments <- attachments_from_ids(data),
-         mentions <- Formatter.parse_mentions(status),
          inReplyTo <- get_replied_to_activity(data["in_reply_to_status_id"]),
-         {to, cc} <- to_for_user_and_mentions(user, mentions, inReplyTo, visibility),
-         tags <- Formatter.parse_tags(status, data),
-         content_html <-
+         {content_html, mentions, tags} <-
            make_content_html(
              status,
-             mentions,
              attachments,
-             tags,
-             get_content_type(data["content_type"]),
-             Enum.member?(
-               [true, "true"],
-               Map.get(
-                 data,
-                 "no_attachment_links",
-                 Pleroma.Config.get([:instance, :no_attachment_links], false)
-               )
-             )
+             data
            ),
+         {to, cc} <- to_for_user_and_mentions(user, mentions, inReplyTo, visibility),
          context <- make_context(inReplyTo),
          cw <- data["spoiler_text"],
          full_payload <- String.trim(status <> (data["spoiler_text"] || "")),
@@ -247,7 +227,7 @@ defmodule Pleroma.Web.CommonAPI do
   def report(user, data) do
     with {:account_id, %{"account_id" => account_id}} <- {:account_id, data},
          {:account, %User{} = account} <- {:account, User.get_by_id(account_id)},
-         {:ok, content_html} <- make_report_content_html(data["comment"]),
+         {:ok, {content_html, _, _}} <- make_report_content_html(data["comment"]),
          {:ok, statuses} <- get_report_statuses(account, data),
          {:ok, activity} <-
            ActivityPub.flag(%{
diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex
index 1d3a314ce..20123854d 100644
--- a/lib/pleroma/web/common_api/utils.ex
+++ b/lib/pleroma/web/common_api/utils.ex
@@ -10,7 +10,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do
   alias Pleroma.Object
   alias Pleroma.Repo
   alias Pleroma.User
-  alias Pleroma.Web
+  alias Pleroma.Config
   alias Pleroma.Web.Endpoint
   alias Pleroma.Web.MediaProxy
   alias Pleroma.Web.ActivityPub.Utils
@@ -100,24 +100,45 @@ defmodule Pleroma.Web.CommonAPI.Utils do
 
   def make_content_html(
         status,
-        mentions,
         attachments,
-        tags,
-        content_type,
-        no_attachment_links \\ false
+        data
       ) do
+    no_attachment_links =
+      data
+      |> Map.get("no_attachment_links", Config.get([:instance, :no_attachment_links]))
+      |> Kernel.in([true, "true"])
+
+    content_type = get_content_type(data["content_type"])
+
     status
-    |> format_input(mentions, tags, content_type)
+    |> format_input(content_type)
     |> maybe_add_attachments(attachments, no_attachment_links)
+    |> maybe_add_nsfw_tag(data)
   end
 
+  defp get_content_type(content_type) do
+    if Enum.member?(Config.get([:instance, :allowed_post_formats]), content_type) do
+      content_type
+    else
+      "text/plain"
+    end
+  end
+
+  defp maybe_add_nsfw_tag({text, mentions, tags}, %{"sensitive" => sensitive})
+       when sensitive in [true, "True", "true", "1"] do
+    {text, mentions, [{"#nsfw", "nsfw"} | tags]}
+  end
+
+  defp maybe_add_nsfw_tag(data, _), do: data
+
   def make_context(%Activity{data: %{"context" => context}}), do: context
   def make_context(_), do: Utils.generate_context_id()
 
-  def maybe_add_attachments(text, _attachments, true = _no_links), do: text
+  def maybe_add_attachments(parsed, _attachments, true = _no_links), do: parsed
 
-  def maybe_add_attachments(text, attachments, _no_links) do
-    add_attachments(text, attachments)
+  def maybe_add_attachments({text, mentions, tags}, attachments, _no_links) do
+    text = add_attachments(text, attachments)
+    {text, mentions, tags}
   end
 
   def add_attachments(text, attachments) do
@@ -135,56 +156,39 @@ defmodule Pleroma.Web.CommonAPI.Utils do
     Enum.join([text | attachment_text], "<br>")
   end
 
-  def format_input(text, mentions, tags, format, options \\ [])
+  def format_input(text, format, options \\ [])
 
   @doc """
   Formatting text to plain text.
   """
-  def format_input(text, mentions, tags, "text/plain", options) do
+  def format_input(text, "text/plain", options) do
     text
     |> Formatter.html_escape("text/plain")
-    |> String.replace(~r/\r?\n/, "<br>")
-    |> (&{[], &1}).()
-    |> Formatter.add_links()
-    |> Formatter.add_user_links(mentions, options[:user_links] || [])
-    |> Formatter.add_hashtag_links(tags)
-    |> Formatter.finalize()
+    |> Formatter.linkify(options)
+    |> (fn {text, mentions, tags} ->
+          {String.replace(text, ~r/\r?\n/, "<br>"), mentions, tags}
+        end).()
   end
 
   @doc """
   Formatting text to html.
   """
-  def format_input(text, mentions, _tags, "text/html", options) do
+  def format_input(text, "text/html", options) do
     text
     |> Formatter.html_escape("text/html")
-    |> (&{[], &1}).()
-    |> Formatter.add_user_links(mentions, options[:user_links] || [])
-    |> Formatter.finalize()
+    |> Formatter.linkify(options)
   end
 
   @doc """
   Formatting text to markdown.
   """
-  def format_input(text, mentions, tags, "text/markdown", options) do
+  def format_input(text, "text/markdown", options) do
+    options = Keyword.put(options, :mentions_escape, true)
+
     text
-    |> Formatter.mentions_escape(mentions)
-    |> Earmark.as_html!()
     |> Formatter.html_escape("text/html")
-    |> (&{[], &1}).()
-    |> Formatter.add_user_links(mentions, options[:user_links] || [])
-    |> Formatter.add_hashtag_links(tags)
-    |> Formatter.finalize()
-  end
-
-  def add_tag_links(text, tags) do
-    tags =
-      tags
-      |> Enum.sort_by(fn {tag, _} -> -String.length(tag) end)
-
-    Enum.reduce(tags, text, fn {full, tag}, text ->
-      url = "<a href='#{Web.base_url()}/tag/#{tag}' rel='tag'>##{tag}</a>"
-      String.replace(text, full, url)
-    end)
+    |> Formatter.linkify(options)
+    |> (fn {text, mentions, tags} -> {Earmark.as_html!(text), mentions, tags} end).()
   end
 
   def make_note_data(
@@ -323,13 +327,13 @@ defmodule Pleroma.Web.CommonAPI.Utils do
 
   def maybe_extract_mentions(_), do: []
 
-  def make_report_content_html(nil), do: {:ok, nil}
+  def make_report_content_html(nil), do: {:ok, {nil, [], []}}
 
   def make_report_content_html(comment) do
     max_size = Pleroma.Config.get([:instance, :max_report_comment_size], 1000)
 
     if String.length(comment) <= max_size do
-      {:ok, format_input(comment, [], [], "text/plain")}
+      {:ok, format_input(comment, "text/plain")}
     else
       {:error, "Comment must be up to #{max_size} characters"}
     end
diff --git a/mix.exs b/mix.exs
index d78825769..5392d94d1 100644
--- a/mix.exs
+++ b/mix.exs
@@ -90,7 +90,10 @@ defmodule Pleroma.Mixfile do
       {:websocket_client, git: "https://github.com/jeremyong/websocket_client.git", only: :test},
       {:floki, "~> 0.20.0"},
       {:ex_syslogger, github: "slashmili/ex_syslogger", tag: "1.4.0"},
-      {:timex, "~> 3.5"}
+      {:timex, "~> 3.5"},
+      {:auto_linker,
+       git: "https://git.pleroma.social/pleroma/auto_linker.git",
+       ref: "94193ca5f97c1f9fdf3d1469653e2d46fac34bcd"}
     ]
   end
 
diff --git a/mix.lock b/mix.lock
index 5ffaedd16..918702444 100644
--- a/mix.lock
+++ b/mix.lock
@@ -1,4 +1,5 @@
 %{
+  "auto_linker": {:git, "https://git.pleroma.social/pleroma/auto_linker.git", "94193ca5f97c1f9fdf3d1469653e2d46fac34bcd", [ref: "94193ca5f97c1f9fdf3d1469653e2d46fac34bcd"]},
   "base64url": {:hex, :base64url, "0.0.1", "36a90125f5948e3afd7be97662a1504b934dd5dac78451ca6e9abf85a10286be", [:rebar], [], "hexpm"},
   "bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], [], "hexpm"},
   "cachex": {:hex, :cachex, "3.0.2", "1351caa4e26e29f7d7ec1d29b53d6013f0447630bbf382b4fb5d5bad0209f203", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm"},
diff --git a/test/formatter_test.exs b/test/formatter_test.exs
index f14077d25..7d8864bf4 100644
--- a/test/formatter_test.exs
+++ b/test/formatter_test.exs
@@ -21,22 +21,16 @@ defmodule Pleroma.FormatterTest do
       expected_text =
         "I love <a class='hashtag' data-tag='cofe' href='http://localhost:4001/tag/cofe' rel='tag'>#cofe</a> and <a class='hashtag' data-tag='2hu' href='http://localhost:4001/tag/2hu' rel='tag'>#2hu</a>"
 
-      tags = Formatter.parse_tags(text)
-
-      assert expected_text ==
-               Formatter.add_hashtag_links({[], text}, tags) |> Formatter.finalize()
+      assert {^expected_text, [], _tags} = Formatter.linkify(text)
     end
 
     test "does not turn html characters to tags" do
-      text = "Fact #3: pleroma does what mastodon't"
+      text = "#fact_3: pleroma does what mastodon't"
 
       expected_text =
-        "Fact <a class='hashtag' data-tag='3' href='http://localhost:4001/tag/3' rel='tag'>#3</a>: pleroma does what mastodon't"
+        "<a class='hashtag' data-tag='fact_3' href='http://localhost:4001/tag/fact_3' rel='tag'>#fact_3</a>: pleroma does what mastodon't"
 
-      tags = Formatter.parse_tags(text)
-
-      assert expected_text ==
-               Formatter.add_hashtag_links({[], text}, tags) |> Formatter.finalize()
+      assert {^expected_text, [], _tags} = Formatter.linkify(text)
     end
   end
 
@@ -47,79 +41,79 @@ defmodule Pleroma.FormatterTest do
       expected =
         "Hey, check out <a href=\"https://www.youtube.com/watch?v=8Zg1-TufF%20zY?x=1&y=2#blabla\">https://www.youtube.com/watch?v=8Zg1-TufF%20zY?x=1&y=2#blabla</a> ."
 
-      assert Formatter.add_links({[], text}) |> Formatter.finalize() == expected
+      assert {^expected, [], []} = Formatter.linkify(text)
 
       text = "https://mastodon.social/@lambadalambda"
 
       expected =
         "<a href=\"https://mastodon.social/@lambadalambda\">https://mastodon.social/@lambadalambda</a>"
 
-      assert Formatter.add_links({[], text}) |> Formatter.finalize() == expected
+      assert {^expected, [], []} = Formatter.linkify(text)
 
       text = "https://mastodon.social:4000/@lambadalambda"
 
       expected =
         "<a href=\"https://mastodon.social:4000/@lambadalambda\">https://mastodon.social:4000/@lambadalambda</a>"
 
-      assert Formatter.add_links({[], text}) |> Formatter.finalize() == expected
+      assert {^expected, [], []} = Formatter.linkify(text)
 
       text = "@lambadalambda"
       expected = "@lambadalambda"
 
-      assert Formatter.add_links({[], text}) |> Formatter.finalize() == expected
+      assert {^expected, [], []} = Formatter.linkify(text)
 
       text = "http://www.cs.vu.nl/~ast/intel/"
       expected = "<a href=\"http://www.cs.vu.nl/~ast/intel/\">http://www.cs.vu.nl/~ast/intel/</a>"
 
-      assert Formatter.add_links({[], text}) |> Formatter.finalize() == expected
+      assert {^expected, [], []} = Formatter.linkify(text)
 
       text = "https://forum.zdoom.org/viewtopic.php?f=44&t=57087"
 
       expected =
         "<a href=\"https://forum.zdoom.org/viewtopic.php?f=44&t=57087\">https://forum.zdoom.org/viewtopic.php?f=44&t=57087</a>"
 
-      assert Formatter.add_links({[], text}) |> Formatter.finalize() == expected
+      assert {^expected, [], []} = Formatter.linkify(text)
 
       text = "https://en.wikipedia.org/wiki/Sophia_(Gnosticism)#Mythos_of_the_soul"
 
       expected =
         "<a href=\"https://en.wikipedia.org/wiki/Sophia_(Gnosticism)#Mythos_of_the_soul\">https://en.wikipedia.org/wiki/Sophia_(Gnosticism)#Mythos_of_the_soul</a>"
 
-      assert Formatter.add_links({[], text}) |> Formatter.finalize() == expected
+      assert {^expected, [], []} = Formatter.linkify(text)
 
       text = "https://www.google.co.jp/search?q=Nasim+Aghdam"
 
       expected =
         "<a href=\"https://www.google.co.jp/search?q=Nasim+Aghdam\">https://www.google.co.jp/search?q=Nasim+Aghdam</a>"
 
-      assert Formatter.add_links({[], text}) |> Formatter.finalize() == expected
+      assert {^expected, [], []} = Formatter.linkify(text)
 
       text = "https://en.wikipedia.org/wiki/Duff's_device"
 
       expected =
         "<a href=\"https://en.wikipedia.org/wiki/Duff's_device\">https://en.wikipedia.org/wiki/Duff's_device</a>"
 
-      assert Formatter.add_links({[], text}) |> Formatter.finalize() == expected
+      assert {^expected, [], []} = Formatter.linkify(text)
 
       text = "https://pleroma.com https://pleroma.com/sucks"
 
       expected =
         "<a href=\"https://pleroma.com\">https://pleroma.com</a> <a href=\"https://pleroma.com/sucks\">https://pleroma.com/sucks</a>"
 
-      assert Formatter.add_links({[], text}) |> Formatter.finalize() == expected
+      assert {^expected, [], []} = Formatter.linkify(text)
 
       text = "xmpp:contact@hacktivis.me"
 
       expected = "<a href=\"xmpp:contact@hacktivis.me\">xmpp:contact@hacktivis.me</a>"
 
-      assert Formatter.add_links({[], text}) |> Formatter.finalize() == expected
+      assert {^expected, [], []} = Formatter.linkify(text)
 
       text =
         "magnet:?xt=urn:btih:7ec9d298e91d6e4394d1379caf073c77ff3e3136&tr=udp%3A%2F%2Fopentor.org%3A2710&tr=udp%3A%2F%2Ftracker.blackunicorn.xyz%3A6969&tr=udp%3A%2F%2Ftracker.ccc.de%3A80&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969&tr=udp%3A%2F%2Ftracker.openbittorrent.com%3A80&tr=wss%3A%2F%2Ftracker.btorrent.xyz&tr=wss%3A%2F%2Ftracker.fastcast.nz&tr=wss%3A%2F%2Ftracker.openwebtorrent.com"
 
       expected = "<a href=\"#{text}\">#{text}</a>"
 
-      assert Formatter.add_links({[], text}) |> Formatter.finalize() == expected
+      assert {^expected, [], []} = Formatter.linkify(text)
     end
   end
 
@@ -136,12 +130,9 @@ defmodule Pleroma.FormatterTest do
 
       archaeme_remote = insert(:user, %{nickname: "archaeme@archae.me"})
 
-      mentions = Pleroma.Formatter.parse_mentions(text)
+      {text, mentions, []} = Formatter.linkify(text)
 
-      {subs, text} = Formatter.add_user_links({[], text}, mentions)
-
-      assert length(subs) == 3
-      Enum.each(subs, fn {uuid, _} -> assert String.contains?(text, uuid) end)
+      assert length(mentions) == 3
 
       expected_text =
         "<span class='h-card'><a data-user='#{gsimg.id}' class='u-url mention' href='#{
@@ -152,7 +143,7 @@ defmodule Pleroma.FormatterTest do
           archaeme_remote.id
         }' class='u-url mention' href='#{archaeme_remote.ap_id}'>@<span>archaeme</span></a></span>"
 
-      assert expected_text == Formatter.finalize({subs, text})
+      assert expected_text == text
     end
 
     test "gives a replacement for user links when the user is using Osada" do
@@ -160,48 +151,35 @@ defmodule Pleroma.FormatterTest do
 
       text = "@mike@osada.macgirvin.com test"
 
-      mentions = Formatter.parse_mentions(text)
+      {text, mentions, []} = Formatter.linkify(text)
 
-      {subs, text} = Formatter.add_user_links({[], text}, mentions)
-
-      assert length(subs) == 1
-      Enum.each(subs, fn {uuid, _} -> assert String.contains?(text, uuid) end)
+      assert length(mentions) == 1
 
       expected_text =
         "<span class='h-card'><a data-user='#{mike.id}' class='u-url mention' href='#{mike.ap_id}'>@<span>mike</span></a></span> test"
 
-      assert expected_text == Formatter.finalize({subs, text})
+      assert expected_text == text
     end
 
     test "gives a replacement for single-character local nicknames" do
       text = "@o hi"
       o = insert(:user, %{nickname: "o"})
 
-      mentions = Formatter.parse_mentions(text)
+      {text, mentions, []} = Formatter.linkify(text)
 
-      {subs, text} = Formatter.add_user_links({[], text}, mentions)
-
-      assert length(subs) == 1
-      Enum.each(subs, fn {uuid, _} -> assert String.contains?(text, uuid) end)
+      assert length(mentions) == 1
 
       expected_text =
         "<span class='h-card'><a data-user='#{o.id}' class='u-url mention' href='#{o.ap_id}'>@<span>o</span></a></span> hi"
 
-      assert expected_text == Formatter.finalize({subs, text})
+      assert expected_text == text
     end
 
     test "does not give a replacement for single-character local nicknames who don't exist" do
       text = "@a hi"
 
-      mentions = Formatter.parse_mentions(text)
-
-      {subs, text} = Formatter.add_user_links({[], text}, mentions)
-
-      assert Enum.empty?(subs)
-      Enum.each(subs, fn {uuid, _} -> assert String.contains?(text, uuid) end)
-
       expected_text = "@a hi"
-      assert expected_text == Formatter.finalize({subs, text})
+      assert {^expected_text, [] = _mentions, [] = _tags} = Formatter.linkify(text)
     end
   end
 
@@ -209,14 +187,14 @@ defmodule Pleroma.FormatterTest do
     test "parses tags in the text" do
       text = "Here's a #Test. Maybe these are #working or not. What about #漢字? And #は｡"
 
-      expected = [
+      expected_tags = [
         {"#Test", "test"},
         {"#working", "working"},
-        {"#漢字", "漢字"},
-        {"#は", "は"}
+        {"#は", "は"},
+        {"#漢字", "漢字"}
       ]
 
-      assert Formatter.parse_tags(text) == expected
+      assert {_text, [], ^expected_tags} = Formatter.linkify(text)
     end
   end
 
@@ -230,15 +208,15 @@ defmodule Pleroma.FormatterTest do
     archaeme = insert(:user, %{nickname: "archaeme"})
     archaeme_remote = insert(:user, %{nickname: "archaeme@archae.me"})
 
-    expected_result = [
-      {"@gsimg", gsimg},
+    expected_mentions = [
       {"@archaeme", archaeme},
       {"@archaeme@archae.me", archaeme_remote},
-      {"@o", o},
-      {"@jimm", jimm}
+      {"@gsimg", gsimg},
+      {"@jimm", jimm},
+      {"@o", o}
     ]
 
-    assert Formatter.parse_mentions(text) == expected_result
+    assert {_text, ^expected_mentions, []} = Formatter.linkify(text)
   end
 
   test "it adds cool emoji" do
@@ -281,22 +259,10 @@ defmodule Pleroma.FormatterTest do
     assert Formatter.get_emoji(text) == []
   end
 
-  describe "/mentions_escape" do
-    test "it returns text with escaped mention names" do
-      text = """
-      @a_breakin_glass@cybre.space
-      (also, little voice inside my head thinking "maybe this will encourage people
-      pronouncing it properly instead of saying _raKEWdo_ ")
-      """
+  test "it escapes HTML in plain text" do
+    text = "hello & world google.com/?a=b&c=d \n http://test.com/?a=b&c=d 1"
+    expected = "hello &amp; world google.com/?a=b&c=d \n http://test.com/?a=b&c=d 1"
 
-      escape_text = """
-      @a\\_breakin\\_glass@cybre\\.space
-      (also, little voice inside my head thinking \"maybe this will encourage people
-      pronouncing it properly instead of saying _raKEWdo_ \")
-      """
-
-      mentions = [{"@a_breakin_glass@cybre.space", %{}}]
-      assert Formatter.mentions_escape(text, mentions) == escape_text
-    end
+    assert Formatter.html_escape(text, "text/plain") == expected
   end
 end
diff --git a/test/web/common_api/common_api_utils_test.exs b/test/web/common_api/common_api_utils_test.exs
index faed6b685..dc7b4c229 100644
--- a/test/web/common_api/common_api_utils_test.exs
+++ b/test/web/common_api/common_api_utils_test.exs
@@ -57,19 +57,19 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do
     assert expected == Utils.emoji_from_profile(user)
   end
 
-  describe "format_input/4" do
+  describe "format_input/3" do
     test "works for bare text/plain" do
       text = "hello world!"
       expected = "hello world!"
 
-      output = Utils.format_input(text, [], [], "text/plain")
+      {output, [], []} = Utils.format_input(text, "text/plain")
 
       assert output == expected
 
       text = "hello world!\n\nsecond paragraph!"
       expected = "hello world!<br><br>second paragraph!"
 
-      output = Utils.format_input(text, [], [], "text/plain")
+      {output, [], []} = Utils.format_input(text, "text/plain")
 
       assert output == expected
     end
@@ -78,14 +78,14 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do
       text = "<p>hello world!</p>"
       expected = "<p>hello world!</p>"
 
-      output = Utils.format_input(text, [], [], "text/html")
+      {output, [], []} = Utils.format_input(text, "text/html")
 
       assert output == expected
 
       text = "<p>hello world!</p>\n\n<p>second paragraph</p>"
       expected = "<p>hello world!</p>\n\n<p>second paragraph</p>"
 
-      output = Utils.format_input(text, [], [], "text/html")
+      {output, [], []} = Utils.format_input(text, "text/html")
 
       assert output == expected
     end
@@ -94,14 +94,32 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do
       text = "**hello world**"
       expected = "<p><strong>hello world</strong></p>\n"
 
-      output = Utils.format_input(text, [], [], "text/markdown")
+      {output, [], []} = Utils.format_input(text, "text/markdown")
 
       assert output == expected
 
       text = "**hello world**\n\n*another paragraph*"
       expected = "<p><strong>hello world</strong></p>\n<p><em>another paragraph</em></p>\n"
 
-      output = Utils.format_input(text, [], [], "text/markdown")
+      {output, [], []} = Utils.format_input(text, "text/markdown")
+
+      assert output == expected
+    end
+
+    test "works for text/markdown with mentions" do
+      {:ok, user} =
+        UserBuilder.insert(%{nickname: "user__test", ap_id: "http://foo.com/user__test"})
+
+      text = "**hello world**\n\n*another @user__test and @user__test google.com paragraph*"
+
+      expected =
+        "<p><strong>hello world</strong></p>\n<p><em>another <span class='h-card'><a data-user='#{
+          user.id
+        }' class='u-url mention' href='http://foo.com/user__test'>@<span>user__test</span></a></span> and <span class='h-card'><a data-user='#{
+          user.id
+        }' class='u-url mention' href='http://foo.com/user__test'>@<span>user__test</span></a></span> <a href=\"http://google.com\">google.com</a> paragraph</em></p>\n"
+
+      {output, _, _} = Utils.format_input(text, "text/markdown")
 
       assert output == expected
     end

From ed7fd6b47e93e0874e3d79f124b71693e43dbb2c Mon Sep 17 00:00:00 2001
From: Maxim Filippov <colixer@gmail.com>
Date: Wed, 27 Feb 2019 03:08:03 +0300
Subject: [PATCH 16/68] Add missing docs and tests

---
 docs/Admin-API.md                             | 201 ++++++++++++------
 .../admin_api/admin_api_controller_test.exs   |  35 +++
 2 files changed, 168 insertions(+), 68 deletions(-)

diff --git a/docs/Admin-API.md b/docs/Admin-API.md
index 016444d58..508981d38 100644
--- a/docs/Admin-API.md
+++ b/docs/Admin-API.md
@@ -1,108 +1,173 @@
 # Admin API
+
 Authentication is required and the user must be an admin.
 
+## `/api/pleroma/admin/users`
+
+### List users
+
+- Method `GET`
+- Response:
+
+```JSON
+[
+    {
+        "deactivated": bool,
+        "id": integer,
+        "nickname": string
+    },
+    ...
+]
+```
+
 ## `/api/pleroma/admin/user`
+
 ### Remove a user
-* Method `DELETE`
-* Params:
-    * `nickname`
-* Response: User’s nickname
+
+- Method `DELETE`
+- Params:
+  - `nickname`
+- Response: User’s nickname
+
 ### Create a user
-* Method: `POST`
-* Params:
-    * `nickname`
-    * `email`
-    * `password`
-* Response: User’s nickname
 
-## `/api/pleroma/admin/users/tag`
-### Tag a list of users
-* Method: `PUT`
-* Params:
-    * `nickname`
-    * `tags`
-### Untag a list of users
-* Method: `DELETE`
-* Params:
-    * `nickname`
-    * `tags`
+- Method: `POST`
+- Params:
+  - `nickname`
+  - `email`
+  - `password`
+- Response: User’s nickname
+
+## `/api/pleroma/admin/users/:nickname/toggle_activation`
+
+### Toggle user activation
+
+- Method: `PATCH`
+- Params:
+  - `nickname`
+- Response: User’s object
 
-## `/api/pleroma/admin/permission_group/:nickname`
-### Get user user permission groups membership
-* Method: `GET`
-* Params: none
-* Response:
 ```JSON
 {
-	"is_moderator": bool,
-	"is_admin": bool
+    "deactivated": bool,
+    "id": integer,
+    "nickname": string
+}
+```
+
+## `/api/pleroma/admin/users/tag`
+
+### Tag a list of users
+
+- Method: `PUT`
+- Params:
+  - `nickname`
+  - `tags`
+
+### Untag a list of users
+
+- Method: `DELETE`
+- Params:
+  - `nickname`
+  - `tags`
+
+## `/api/pleroma/admin/permission_group/:nickname`
+
+### Get user user permission groups membership
+
+- Method: `GET`
+- Params: none
+- Response:
+
+```JSON
+{
+    "is_moderator": bool,
+    "is_admin": bool
 }
 ```
 
 ## `/api/pleroma/admin/permission_group/:nickname/:permission_group`
+
 Note: Available `:permission_group` is currently moderator and admin. 404 is returned when the permission group doesn’t exist.
 
 ### Get user user permission groups membership
-* Method: `GET`
-* Params: none
-* Response:
+
+- Method: `GET`
+- Params: none
+- Response:
+
 ```JSON
 {
-	"is_moderator": bool,
-	"is_admin": bool
+    "is_moderator": bool,
+    "is_admin": bool
 }
 ```
+
 ### Add user in permission group
-* Method: `POST`
-* Params: none
-* Response:
-    * On failure: ``{"error": "…"}``
-    * On success: JSON of the ``user.info``
+
+- Method: `POST`
+- Params: none
+- Response:
+  - On failure: `{"error": "…"}`
+  - On success: JSON of the `user.info`
+
 ### Remove user from permission group
-* Method: `DELETE`
-* Params: none
-* Response:
-    * On failure: ``{"error": "…"}``
-    * On success: JSON of the ``user.info``
-* Note: An admin cannot revoke their own admin status.
+
+- Method: `DELETE`
+- Params: none
+- Response:
+  - On failure: `{"error": "…"}`
+  - On success: JSON of the `user.info`
+- Note: An admin cannot revoke their own admin status.
 
 ## `/api/pleroma/admin/activation_status/:nickname`
 
 ### Active or deactivate a user
-* Method: `PUT`
-* Params:
-    * `nickname`
-    * `status` BOOLEAN field, false value means deactivation.
+
+- Method: `PUT`
+- Params:
+  - `nickname`
+  - `status` BOOLEAN field, false value means deactivation.
 
 ## `/api/pleroma/admin/relay`
+
 ### Follow a Relay
-* Methods: `POST`
-* Params:
-    * `relay_url`
-* Response:
-    * On success: URL of the followed relay
+
+- Methods: `POST`
+- Params:
+  - `relay_url`
+- Response:
+  - On success: URL of the followed relay
+
 ### Unfollow a Relay
-* Methods: `DELETE`
-* Params:
-    * `relay_url`
-* Response:
-    * On success: URL of the unfollowed relay
+
+- Methods: `DELETE`
+- Params:
+  - `relay_url`
+- Response:
+  - On success: URL of the unfollowed relay
 
 ## `/api/pleroma/admin/invite_token`
+
 ### Get a account registeration invite token
-* Methods: `GET`
-* Params: none
-* Response: invite token (base64 string)
+
+- Methods: `GET`
+- Params: none
+- Response: invite token (base64 string)
 
 ## `/api/pleroma/admin/email_invite`
+
 ### Sends registration invite via email
-* Methods: `POST`
-* Params:
-    * `email`
-    * `name`, optionnal
+
+- Methods: `POST`
+- Params:
+  - `email`
+  - `name`, optionnal
 
 ## `/api/pleroma/admin/password_reset`
+
 ### Get a password reset token for a given nickname
-* Methods: `GET`
-* Params: none
-* Response: password reset token (base64 string)
+
+- Methods: `GET`
+- Params: none
+- Response: password reset token (base64 string)
diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs
index 9fbaaba39..f6ae16844 100644
--- a/test/web/admin_api/admin_api_controller_test.exs
+++ b/test/web/admin_api/admin_api_controller_test.exs
@@ -330,4 +330,39 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
 
     assert conn.status == 200
   end
+
+  test "GET /api/pleroma/admin/users" do
+    admin = insert(:user, info: %{is_admin: true})
+    user = insert(:user)
+
+    conn =
+      build_conn()
+      |> assign(:user, admin)
+      |> get("/api/pleroma/admin/users")
+
+    assert json_response(conn, 200) == [
+             %{
+               "deactivated" => user.info.deactivated,
+               "id" => user.id,
+               "nickname" => user.nickname
+             }
+           ]
+  end
+
+  test "PATCH /api/pleroma/admin/users/:nickname/toggle_activation" do
+    admin = insert(:user, info: %{is_admin: true})
+    user = insert(:user)
+
+    conn =
+      build_conn()
+      |> assign(:user, admin)
+      |> patch("/api/pleroma/admin/users/#{user.nickname}/toggle_activation")
+
+    assert json_response(conn, 200) ==
+             %{
+               "deactivated" => !user.info.deactivated,
+               "id" => user.id,
+               "nickname" => user.nickname
+             }
+  end
 end

From 5d961d536cd190c8201d53624680a6f3384ffd9b Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Wed, 27 Feb 2019 15:40:30 +0700
Subject: [PATCH 17/68] fix formatter

---
 lib/pleroma/formatter.ex                      |  4 ++++
 lib/pleroma/web/common_api/utils.ex           |  2 +-
 test/web/common_api/common_api_utils_test.exs | 18 +++++++++++++++---
 3 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex
index 51d08c5ee..048c032ed 100644
--- a/lib/pleroma/formatter.ex
+++ b/lib/pleroma/formatter.ex
@@ -91,6 +91,10 @@ defmodule Pleroma.Formatter do
 
   def get_emoji(_), do: []
 
+  def html_escape({text, mentions, hashtags}, type) do
+    {html_escape(text, type), mentions, hashtags}
+  end
+
   def html_escape(text, "text/html") do
     HTML.filter_tags(text)
   end
diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex
index 20123854d..e4b9102c5 100644
--- a/lib/pleroma/web/common_api/utils.ex
+++ b/lib/pleroma/web/common_api/utils.ex
@@ -186,9 +186,9 @@ defmodule Pleroma.Web.CommonAPI.Utils do
     options = Keyword.put(options, :mentions_escape, true)
 
     text
-    |> Formatter.html_escape("text/html")
     |> Formatter.linkify(options)
     |> (fn {text, mentions, tags} -> {Earmark.as_html!(text), mentions, tags} end).()
+    |> Formatter.html_escape("text/html")
   end
 
   def make_note_data(
diff --git a/test/web/common_api/common_api_utils_test.exs b/test/web/common_api/common_api_utils_test.exs
index dc7b4c229..684f2a23f 100644
--- a/test/web/common_api/common_api_utils_test.exs
+++ b/test/web/common_api/common_api_utils_test.exs
@@ -104,6 +104,18 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do
       {output, [], []} = Utils.format_input(text, "text/markdown")
 
       assert output == expected
+
+      text = """
+      > cool quote
+
+      by someone
+      """
+
+      expected = "<blockquote><p>cool quote</p>\n</blockquote>\n<p>by someone</p>\n"
+
+      {output, [], []} = Utils.format_input(text, "text/markdown")
+
+      assert output == expected
     end
 
     test "works for text/markdown with mentions" do
@@ -113,11 +125,11 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do
       text = "**hello world**\n\n*another @user__test and @user__test google.com paragraph*"
 
       expected =
-        "<p><strong>hello world</strong></p>\n<p><em>another <span class='h-card'><a data-user='#{
+        "<p><strong>hello world</strong></p>\n<p><em>another <span class=\"h-card\"><a data-user=\"#{
           user.id
-        }' class='u-url mention' href='http://foo.com/user__test'>@<span>user__test</span></a></span> and <span class='h-card'><a data-user='#{
+        }\" class=\"u-url mention\" href=\"http://foo.com/user__test\">@<span>user__test</span></a></span> and <span class=\"h-card\"><a data-user=\"#{
           user.id
-        }' class='u-url mention' href='http://foo.com/user__test'>@<span>user__test</span></a></span> <a href=\"http://google.com\">google.com</a> paragraph</em></p>\n"
+        }\" class=\"u-url mention\" href=\"http://foo.com/user__test\">@<span>user__test</span></a></span> <a href=\"http://google.com\">google.com</a> paragraph</em></p>\n"
 
       {output, _, _} = Utils.format_input(text, "text/markdown")
 

From 153664096255208055ae2e0b31ea20238ad540b2 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Wed, 27 Feb 2019 13:01:10 +0000
Subject: [PATCH 18/68] mastodon api: embed relationship card under account
 card for Pleroma FE convenience

---
 .../web/mastodon_api/views/account_view.ex    | 11 +++-
 test/web/mastodon_api/account_view_test.exs   | 66 ++++++++++++++++++-
 2 files changed, 73 insertions(+), 4 deletions(-)

diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex
index 8fdefdebd..c32f27be2 100644
--- a/lib/pleroma/web/mastodon_api/views/account_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/account_view.ex
@@ -32,7 +32,11 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
     }
   end
 
-  def render("relationship.json", %{user: user, target: target}) do
+  def render("relationship.json", %{user: nil, target: _target}) do
+    %{}
+  end
+
+  def render("relationship.json", %{user: %User{} = user, target: %User{} = target}) do
     follow_activity = Pleroma.Web.ActivityPub.Utils.fetch_latest_follow(user, target)
 
     requested =
@@ -85,6 +89,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
 
     bio = HTML.filter_tags(user.bio, User.html_filter_policy(opts[:for]))
 
+    relationship = render("relationship.json", %{user: opts[:for], target: user})
+
     %{
       id: to_string(user.id),
       username: username_from_nickname(user.nickname),
@@ -115,7 +121,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
         confirmation_pending: user_info.confirmation_pending,
         tags: user.tags,
         is_moderator: user.info.is_moderator,
-        is_admin: user.info.is_admin
+        is_admin: user.info.is_admin,
+        relationship: relationship
       }
     }
   end
diff --git a/test/web/mastodon_api/account_view_test.exs b/test/web/mastodon_api/account_view_test.exs
index f8cd68173..6be66ef63 100644
--- a/test/web/mastodon_api/account_view_test.exs
+++ b/test/web/mastodon_api/account_view_test.exs
@@ -63,7 +63,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do
         confirmation_pending: false,
         tags: [],
         is_admin: false,
-        is_moderator: false
+        is_moderator: false,
+        relationship: %{}
       }
     }
 
@@ -106,7 +107,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do
         confirmation_pending: false,
         tags: [],
         is_admin: false,
-        is_moderator: false
+        is_moderator: false,
+        relationship: %{}
       }
     }
 
@@ -148,4 +150,64 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do
 
     assert expected == AccountView.render("relationship.json", %{user: user, target: other_user})
   end
+
+  test "represent an embedded relationship" do
+    user =
+      insert(:user, %{
+        info: %{note_count: 5, follower_count: 3, source_data: %{"type" => "Service"}},
+        nickname: "shp@shitposter.club",
+        inserted_at: ~N[2017-08-15 15:47:06.597036]
+      })
+
+    other_user = insert(:user)
+
+    {:ok, other_user} = User.follow(other_user, user)
+    {:ok, other_user} = User.block(other_user, user)
+
+    expected = %{
+      id: to_string(user.id),
+      username: "shp",
+      acct: user.nickname,
+      display_name: user.name,
+      locked: false,
+      created_at: "2017-08-15T15:47:06.000Z",
+      followers_count: 3,
+      following_count: 0,
+      statuses_count: 5,
+      note: user.bio,
+      url: user.ap_id,
+      avatar: "http://localhost:4001/images/avi.png",
+      avatar_static: "http://localhost:4001/images/avi.png",
+      header: "http://localhost:4001/images/banner.png",
+      header_static: "http://localhost:4001/images/banner.png",
+      emojis: [],
+      fields: [],
+      bot: true,
+      source: %{
+        note: "",
+        privacy: "public",
+        sensitive: false
+      },
+      pleroma: %{
+        confirmation_pending: false,
+        tags: [],
+        is_admin: false,
+        is_moderator: false,
+        relationship: %{
+          id: to_string(user.id),
+          following: false,
+          followed_by: false,
+          blocking: true,
+          muting: false,
+          muting_notifications: false,
+          requested: false,
+          domain_blocking: false,
+          showing_reblogs: false,
+          endorsed: false
+        }
+      }
+    }
+
+    assert expected == AccountView.render("account.json", %{user: user, for: other_user})
+  end
 end

From 8d8cb08f94490299bfc7fe97381a34e4a7a095a9 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 27 Feb 2019 14:51:07 +0100
Subject: [PATCH 19/68] Add follow request test.

---
 test/user_test.exs | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/test/user_test.exs b/test/user_test.exs
index 0b1c39ecf..cbe4693fc 100644
--- a/test/user_test.exs
+++ b/test/user_test.exs
@@ -50,6 +50,20 @@ defmodule Pleroma.UserTest do
     assert expected_followers_collection == User.ap_followers(user)
   end
 
+  test "returns all pending follow requests" do
+    unlocked = insert(:user)
+    locked = insert(:user, %{info: %{locked: true}})
+    follower = insert(:user)
+
+    Pleroma.Web.TwitterAPI.TwitterAPI.follow(follower, %{"user_id" => unlocked.id})
+    Pleroma.Web.TwitterAPI.TwitterAPI.follow(follower, %{"user_id" => locked.id})
+
+    assert {:ok, []} = User.get_follow_requests(unlocked)
+    assert {:ok, [activity]} = User.get_follow_requests(locked)
+
+    assert activity
+  end
+
   test "follow_all follows mutliple users" do
     user = insert(:user)
     followed_zero = insert(:user)

From b24cc44e8d1e1b29cdeb916f192e30303a55ff5d Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 27 Feb 2019 15:01:54 +0100
Subject: [PATCH 20/68] Follower requests: Utilize object index.

Closes #677
---
 lib/pleroma/user.ex | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 01d532ab3..c5085fa82 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -613,9 +613,10 @@ defmodule Pleroma.User do
         ),
       where:
         fragment(
-          "? @> ?",
+          "coalesce((?)->'object'->>'id', (?)->>'object') = ?",
           a.data,
-          ^%{"object" => user.ap_id}
+          a.data,
+          ^user.ap_id
         )
     )
   end

From c4235f96bde49def1fb6927ba79a49a0f75cd60a Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 27 Feb 2019 16:37:42 +0100
Subject: [PATCH 21/68] Add `with_muted` param.

---
 docs/Differences-in-MastodonAPI-Responses.md | 4 ++++
 lib/pleroma/web/activity_pub/activity_pub.ex | 2 ++
 test/web/activity_pub/activity_pub_test.exs  | 7 +++++++
 3 files changed, 13 insertions(+)

diff --git a/docs/Differences-in-MastodonAPI-Responses.md b/docs/Differences-in-MastodonAPI-Responses.md
index f6a5b6461..3026e1173 100644
--- a/docs/Differences-in-MastodonAPI-Responses.md
+++ b/docs/Differences-in-MastodonAPI-Responses.md
@@ -9,3 +9,7 @@ Pleroma uses 128-bit ids as opposed to Mastodon's 64 bits. However just like Mas
 ## Attachment cap
 
 Some apps operate under the assumption that no more than 4 attachments can be returned or uploaded. Pleroma however does not enforce any limits on attachment count neither when returning the status object nor when posting.
+
+## Timelines
+
+Adding the parameter `with_muted=true` to the timeline queries will also return activities by muted (not by blocked!) users.
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index cc255cc9e..52404c7e5 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -602,6 +602,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp restrict_reblogs(query, _), do: query
 
+  defp restrict_muted(query, %{"with_muted" => val}) when val in [true, "true", "1"], do: query
+
   defp restrict_muted(query, %{"muting_user" => %User{info: info}}) do
     mutes = info.mutes
 
diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs
index 11262c523..ac3a565de 100644
--- a/test/web/activity_pub/activity_pub_test.exs
+++ b/test/web/activity_pub/activity_pub_test.exs
@@ -291,6 +291,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
     assert Enum.member?(activities, activity_three)
     refute Enum.member?(activities, activity_one)
 
+    # Calling with 'with_muted' will deliver muted activities, too.
+    activities = ActivityPub.fetch_activities([], %{"muting_user" => user, "with_muted" => true})
+
+    assert Enum.member?(activities, activity_two)
+    assert Enum.member?(activities, activity_three)
+    assert Enum.member?(activities, activity_one)
+
     {:ok, user} = User.unmute(user, %User{ap_id: activity_one.data["actor"]})
 
     activities = ActivityPub.fetch_activities([], %{"muting_user" => user})

From c1ae495878cc95d09a051cc56e2bf7dc7ed0a032 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 27 Feb 2019 16:46:47 +0100
Subject: [PATCH 22/68] Add user muted status info to MastodonAPI.

---
 lib/pleroma/user.ex                              |  1 +
 .../web/mastodon_api/views/status_view.ex        |  2 +-
 test/web/mastodon_api/status_view_test.exs       | 16 ++++++++++++++++
 3 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 01d532ab3..3a8a51e33 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -960,6 +960,7 @@ defmodule Pleroma.User do
     update_and_set_cache(cng)
   end
 
+  def mutes?(nil, _), do: false
   def mutes?(user, %{ap_id: ap_id}), do: Enum.member?(user.info.mutes, ap_id)
 
   def blocks?(user, %{ap_id: ap_id}) do
diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex
index b90e4252a..3468c0e1c 100644
--- a/lib/pleroma/web/mastodon_api/views/status_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/status_view.ex
@@ -168,7 +168,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
       reblogged: present?(repeated),
       favourited: present?(favorited),
       bookmarked: present?(bookmarked),
-      muted: CommonAPI.thread_muted?(user, activity),
+      muted: CommonAPI.thread_muted?(user, activity) || User.mutes?(opts[:for], user),
       pinned: pinned?(activity, user),
       sensitive: sensitive,
       spoiler_text: object["summary"] || "",
diff --git a/test/web/mastodon_api/status_view_test.exs b/test/web/mastodon_api/status_view_test.exs
index 3412a6be2..351dbf673 100644
--- a/test/web/mastodon_api/status_view_test.exs
+++ b/test/web/mastodon_api/status_view_test.exs
@@ -126,6 +126,22 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do
     assert status == expected
   end
 
+  test "tells if the message is muted for some reason" do
+    user = insert(:user)
+    other_user = insert(:user)
+
+    {:ok, user} = User.mute(user, other_user)
+
+    {:ok, activity} = CommonAPI.post(other_user, %{"status" => "test"})
+    status = StatusView.render("status.json", %{activity: activity})
+
+    assert status.muted == false
+
+    status = StatusView.render("status.json", %{activity: activity, for: user})
+
+    assert status.muted == true
+  end
+
   test "a reply" do
     note = insert(:note_activity)
     user = insert(:user)

From 9ade1242c20acae5d27785deb833b453628b12ee Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 27 Feb 2019 16:52:03 +0100
Subject: [PATCH 23/68] Add user muted status info to twitterapi.

---
 .../web/twitter_api/views/activity_view.ex    |  4 +++-
 .../twitter_api/views/activity_view_test.exs  | 19 ++++++++++++++++++-
 2 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/web/twitter_api/views/activity_view.ex b/lib/pleroma/web/twitter_api/views/activity_view.ex
index 661022afa..02ca4ee42 100644
--- a/lib/pleroma/web/twitter_api/views/activity_view.ex
+++ b/lib/pleroma/web/twitter_api/views/activity_view.ex
@@ -10,6 +10,7 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do
   alias Pleroma.Object
   alias Pleroma.Repo
   alias Pleroma.User
+  alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.CommonAPI.Utils
   alias Pleroma.Web.MastodonAPI.StatusView
   alias Pleroma.Web.TwitterAPI.ActivityView
@@ -309,7 +310,8 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do
       "visibility" => StatusView.get_visibility(object),
       "summary" => summary,
       "summary_html" => summary |> Formatter.emojify(object["emoji"]),
-      "card" => card
+      "card" => card,
+      "muted" => CommonAPI.thread_muted?(user, activity) || User.mutes?(opts[:for], user)
     }
   end
 
diff --git a/test/web/twitter_api/views/activity_view_test.exs b/test/web/twitter_api/views/activity_view_test.exs
index 4f854ecaa..0a5384f34 100644
--- a/test/web/twitter_api/views/activity_view_test.exs
+++ b/test/web/twitter_api/views/activity_view_test.exs
@@ -56,6 +56,22 @@ defmodule Pleroma.Web.TwitterAPI.ActivityViewTest do
     assert result["user"]["id"] == user.id
   end
 
+  test "tells if the message is muted for some reason" do
+    user = insert(:user)
+    other_user = insert(:user)
+
+    {:ok, user} = User.mute(user, other_user)
+
+    {:ok, activity} = CommonAPI.post(other_user, %{"status" => "test"})
+    status = ActivityView.render("activity.json", %{activity: activity})
+
+    assert status["muted"] == false
+
+    status = ActivityView.render("activity.json", %{activity: activity, for: user})
+
+    assert status["muted"] == true
+  end
+
   test "a create activity with a html status" do
     text = """
     #Bike log - Commute Tuesday\nhttps://pla.bike/posts/20181211/\n#cycling #CHScycling #commute\nMVIMG_20181211_054020.jpg
@@ -149,7 +165,8 @@ defmodule Pleroma.Web.TwitterAPI.ActivityViewTest do
       "uri" => activity.data["object"]["id"],
       "user" => UserView.render("show.json", %{user: user}),
       "visibility" => "direct",
-      "card" => nil
+      "card" => nil,
+      "muted" => false
     }
 
     assert result == expected

From bbbdbec4fd8f14aa039d7f4a42215544cd6e4932 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 27 Feb 2019 17:24:51 +0100
Subject: [PATCH 24/68] Remove parts of the old activity view.

Not used anymore.
---
 .../representers/activity_representer.ex      | 247 +-----------------
 .../activity_representer_test.exs             |  43 +--
 2 files changed, 6 insertions(+), 284 deletions(-)

diff --git a/lib/pleroma/web/twitter_api/representers/activity_representer.ex b/lib/pleroma/web/twitter_api/representers/activity_representer.ex
index 192ab7334..55c612ddd 100644
--- a/lib/pleroma/web/twitter_api/representers/activity_representer.ex
+++ b/lib/pleroma/web/twitter_api/representers/activity_representer.ex
@@ -6,247 +6,10 @@
 # THIS MODULE IS DEPRECATED! DON'T USE IT!
 # USE THE Pleroma.Web.TwitterAPI.Views.ActivityView MODULE!
 defmodule Pleroma.Web.TwitterAPI.Representers.ActivityRepresenter do
-  use Pleroma.Web.TwitterAPI.Representers.BaseRepresenter
-  alias Pleroma.Web.TwitterAPI.Representers.ObjectRepresenter
-  alias Pleroma.Activity
-  alias Pleroma.Formatter
-  alias Pleroma.HTML
-  alias Pleroma.User
-  alias Pleroma.Web.TwitterAPI.ActivityView
-  alias Pleroma.Web.TwitterAPI.TwitterAPI
-  alias Pleroma.Web.TwitterAPI.UserView
-  alias Pleroma.Web.CommonAPI.Utils
-  alias Pleroma.Web.MastodonAPI.StatusView
-
-  defp user_by_ap_id(user_list, ap_id) do
-    Enum.find(user_list, fn %{ap_id: user_id} -> ap_id == user_id end)
-  end
-
-  def to_map(
-        %Activity{data: %{"type" => "Announce", "actor" => actor, "published" => created_at}} =
-          activity,
-        %{users: users, announced_activity: announced_activity} = opts
-      ) do
-    user = user_by_ap_id(users, actor)
-    created_at = created_at |> Utils.date_to_asctime()
-
-    text = "#{user.nickname} retweeted a status."
-
-    announced_user = user_by_ap_id(users, announced_activity.data["actor"])
-    retweeted_status = to_map(announced_activity, Map.merge(%{user: announced_user}, opts))
-
-    %{
-      "id" => activity.id,
-      "user" => UserView.render("show.json", %{user: user, for: opts[:for]}),
-      "statusnet_html" => text,
-      "text" => text,
-      "is_local" => activity.local,
-      "is_post_verb" => false,
-      "uri" => "tag:#{activity.data["id"]}:objectType=note",
-      "created_at" => created_at,
-      "retweeted_status" => retweeted_status,
-      "statusnet_conversation_id" => conversation_id(announced_activity),
-      "external_url" => activity.data["id"],
-      "activity_type" => "repeat"
-    }
-  end
-
-  def to_map(
-        %Activity{data: %{"type" => "Like", "published" => created_at}} = activity,
-        %{user: user, liked_activity: liked_activity} = opts
-      ) do
-    created_at = created_at |> Utils.date_to_asctime()
-
-    text = "#{user.nickname} favorited a status."
-
-    %{
-      "id" => activity.id,
-      "user" => UserView.render("show.json", %{user: user, for: opts[:for]}),
-      "statusnet_html" => text,
-      "text" => text,
-      "is_local" => activity.local,
-      "is_post_verb" => false,
-      "uri" => "tag:#{activity.data["id"]}:objectType=Favourite",
-      "created_at" => created_at,
-      "in_reply_to_status_id" => liked_activity.id,
-      "external_url" => activity.data["id"],
-      "activity_type" => "like"
-    }
-  end
-
-  def to_map(
-        %Activity{data: %{"type" => "Follow", "object" => followed_id}} = activity,
-        %{user: user} = opts
-      ) do
-    created_at = activity.data["published"] || DateTime.to_iso8601(activity.inserted_at)
-    created_at = created_at |> Utils.date_to_asctime()
-
-    followed = User.get_cached_by_ap_id(followed_id)
-    text = "#{user.nickname} started following #{followed.nickname}"
-
-    %{
-      "id" => activity.id,
-      "user" => UserView.render("show.json", %{user: user, for: opts[:for]}),
-      "attentions" => [],
-      "statusnet_html" => text,
-      "text" => text,
-      "is_local" => activity.local,
-      "is_post_verb" => false,
-      "created_at" => created_at,
-      "in_reply_to_status_id" => nil,
-      "external_url" => activity.data["id"],
-      "activity_type" => "follow"
-    }
-  end
-
-  # TODO:
-  # Make this more proper. Just a placeholder to not break the frontend.
-  def to_map(
-        %Activity{
-          data: %{"type" => "Undo", "published" => created_at, "object" => undid_activity}
-        } = activity,
-        %{user: user} = opts
-      ) do
-    created_at = created_at |> Utils.date_to_asctime()
-
-    text = "#{user.nickname} undid the action at #{undid_activity["id"]}"
-
-    %{
-      "id" => activity.id,
-      "user" => UserView.render("show.json", %{user: user, for: opts[:for]}),
-      "attentions" => [],
-      "statusnet_html" => text,
-      "text" => text,
-      "is_local" => activity.local,
-      "is_post_verb" => false,
-      "created_at" => created_at,
-      "in_reply_to_status_id" => nil,
-      "external_url" => activity.data["id"],
-      "activity_type" => "undo"
-    }
-  end
-
-  def to_map(
-        %Activity{data: %{"type" => "Delete", "published" => created_at, "object" => _}} =
-          activity,
-        %{user: user} = opts
-      ) do
-    created_at = created_at |> Utils.date_to_asctime()
-
-    %{
-      "id" => activity.id,
-      "uri" => activity.data["object"],
-      "user" => UserView.render("show.json", %{user: user, for: opts[:for]}),
-      "attentions" => [],
-      "statusnet_html" => "deleted notice {{tag",
-      "text" => "deleted notice {{tag",
-      "is_local" => activity.local,
-      "is_post_verb" => false,
-      "created_at" => created_at,
-      "in_reply_to_status_id" => nil,
-      "external_url" => activity.data["id"],
-      "activity_type" => "delete"
-    }
-  end
-
-  def to_map(
-        %Activity{data: %{"object" => %{"content" => _content} = object}} = activity,
-        %{user: user} = opts
-      ) do
-    created_at = object["published"] |> Utils.date_to_asctime()
-    like_count = object["like_count"] || 0
-    announcement_count = object["announcement_count"] || 0
-    favorited = opts[:for] && opts[:for].ap_id in (object["likes"] || [])
-    repeated = opts[:for] && opts[:for].ap_id in (object["announcements"] || [])
-    pinned = activity.id in user.info.pinned_activities
-
-    mentions = opts[:mentioned] || []
-
-    attentions =
-      []
-      |> Utils.maybe_notify_to_recipients(activity)
-      |> Utils.maybe_notify_mentioned_recipients(activity)
-      |> Enum.map(fn ap_id -> Enum.find(mentions, fn user -> ap_id == user.ap_id end) end)
-      |> Enum.filter(& &1)
-      |> Enum.map(fn user -> UserView.render("show.json", %{user: user, for: opts[:for]}) end)
-
-    conversation_id = conversation_id(activity)
-
-    tags = activity.data["object"]["tag"] || []
-    possibly_sensitive = activity.data["object"]["sensitive"] || Enum.member?(tags, "nsfw")
-
-    tags = if possibly_sensitive, do: Enum.uniq(["nsfw" | tags]), else: tags
-
-    {_summary, content} = ActivityView.render_content(object)
-
-    html =
-      HTML.filter_tags(content, User.html_filter_policy(opts[:for]))
-      |> Formatter.emojify(object["emoji"])
-
-    attachments = object["attachment"] || []
-
-    reply_parent = Activity.get_in_reply_to_activity(activity)
-
-    reply_user = reply_parent && User.get_cached_by_ap_id(reply_parent.actor)
-
-    summary = HTML.strip_tags(object["summary"])
-
-    card =
-      StatusView.render(
-        "card.json",
-        Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
-      )
-
-    %{
-      "id" => activity.id,
-      "uri" => activity.data["object"]["id"],
-      "user" => UserView.render("show.json", %{user: user, for: opts[:for]}),
-      "statusnet_html" => html,
-      "text" => HTML.strip_tags(content),
-      "is_local" => activity.local,
-      "is_post_verb" => true,
-      "created_at" => created_at,
-      "in_reply_to_status_id" => object["inReplyToStatusId"],
-      "in_reply_to_screen_name" => reply_user && reply_user.nickname,
-      "in_reply_to_profileurl" => User.profile_url(reply_user),
-      "in_reply_to_ostatus_uri" => reply_user && reply_user.ap_id,
-      "in_reply_to_user_id" => reply_user && reply_user.id,
-      "statusnet_conversation_id" => conversation_id,
-      "attachments" => attachments |> ObjectRepresenter.enum_to_list(opts),
-      "attentions" => attentions,
-      "fave_num" => like_count,
-      "repeat_num" => announcement_count,
-      "favorited" => to_boolean(favorited),
-      "repeated" => to_boolean(repeated),
-      "pinned" => pinned,
-      "external_url" => object["external_url"] || object["id"],
-      "tags" => tags,
-      "activity_type" => "post",
-      "possibly_sensitive" => possibly_sensitive,
-      "visibility" => Pleroma.Web.MastodonAPI.StatusView.get_visibility(object),
-      "summary" => summary,
-      "summary_html" => summary |> Formatter.emojify(object["emoji"]),
-      "card" => card
-    }
-  end
-
-  def conversation_id(activity) do
-    with context when not is_nil(context) <- activity.data["context"] do
-      TwitterAPI.context_to_conversation_id(context)
-    else
-      _e -> nil
-    end
-  end
-
-  defp to_boolean(false) do
-    false
-  end
-
-  defp to_boolean(nil) do
-    false
-  end
-
-  defp to_boolean(_) do
-    true
+  def to_map(activity, opts) do
+    Pleroma.Web.TwitterAPI.ActivityView.render(
+      "activity.json",
+      Map.put(opts, :activity, activity)
+    )
   end
 end
diff --git a/test/web/twitter_api/representers/activity_representer_test.exs b/test/web/twitter_api/representers/activity_representer_test.exs
index 365c7f659..0e554623c 100644
--- a/test/web/twitter_api/representers/activity_representer_test.exs
+++ b/test/web/twitter_api/representers/activity_representer_test.exs
@@ -13,36 +13,6 @@ defmodule Pleroma.Web.TwitterAPI.Representers.ActivityRepresenterTest do
   alias Pleroma.Web.TwitterAPI.UserView
   import Pleroma.Factory
 
-  test "an announce activity" do
-    user = insert(:user)
-    note_activity = insert(:note_activity)
-    activity_actor = Repo.get_by(User, ap_id: note_activity.data["actor"])
-    object = Object.get_by_ap_id(note_activity.data["object"]["id"])
-
-    {:ok, announce_activity, _object} = ActivityPub.announce(user, object)
-    note_activity = Activity.get_by_ap_id(note_activity.data["id"])
-
-    status =
-      ActivityRepresenter.to_map(announce_activity, %{
-        users: [user, activity_actor],
-        announced_activity: note_activity,
-        for: user
-      })
-
-    assert status["id"] == announce_activity.id
-    assert status["user"] == UserView.render("show.json", %{user: user, for: user})
-
-    retweeted_status =
-      ActivityRepresenter.to_map(note_activity, %{user: activity_actor, for: user})
-
-    assert retweeted_status["repeated"] == true
-    assert retweeted_status["id"] == note_activity.id
-    assert status["statusnet_conversation_id"] == retweeted_status["statusnet_conversation_id"]
-
-    assert status["retweeted_status"] == retweeted_status
-    assert status["activity_type"] == "repeat"
-  end
-
   test "a like activity" do
     user = insert(:user)
     note_activity = insert(:note_activity)
@@ -168,6 +138,7 @@ defmodule Pleroma.Web.TwitterAPI.Representers.ActivityRepresenterTest do
       "uri" => activity.data["object"]["id"],
       "visibility" => "direct",
       "card" => nil,
+      "muted" => false,
       "summary" => "2hu :2hu:",
       "summary_html" =>
         "2hu <img height=\"32px\" width=\"32px\" alt=\"2hu\" title=\"2hu\" src=\"corndog.png\" />"
@@ -180,18 +151,6 @@ defmodule Pleroma.Web.TwitterAPI.Representers.ActivityRepresenterTest do
            }) == expected_status
   end
 
-  test "an undo for a follow" do
-    follower = insert(:user)
-    followed = insert(:user)
-
-    {:ok, _follow} = ActivityPub.follow(follower, followed)
-    {:ok, unfollow} = ActivityPub.unfollow(follower, followed)
-
-    map = ActivityRepresenter.to_map(unfollow, %{user: follower})
-    assert map["is_post_verb"] == false
-    assert map["activity_type"] == "undo"
-  end
-
   test "a delete activity" do
     object = insert(:note)
     user = User.get_by_ap_id(object.data["actor"])

From 2883f75a3a25599c6217460133578cddcd34ebb4 Mon Sep 17 00:00:00 2001
From: Maxim Filippov <colixer@gmail.com>
Date: Thu, 28 Feb 2019 01:11:56 +0300
Subject: [PATCH 25/68] Add pagination to users admin API

---
 lib/pleroma/user.ex                           | 21 ++++++++++++++++---
 .../web/admin_api/admin_api_controller.ex     | 20 +++++++++++++-----
 .../web/twitter_api/views/user_view.ex        |  9 +++++---
 .../admin_api/admin_api_controller_test.exs   | 20 +++++++++++-------
 4 files changed, 51 insertions(+), 19 deletions(-)

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 01d532ab3..80e4ae296 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -772,10 +772,25 @@ defmodule Pleroma.User do
     Enum.uniq_by(fts_results ++ trigram_results, & &1.id)
   end
 
-  def all_except_one(user) do
-    query = from(u in User, where: u.id != ^user.id)
+  def all_except_one(user, page, page_size) do
+    from(
+      u in User,
+      where: u.id != ^user.id,
+      limit: ^page_size,
+      offset: ^((page - 1) * page_size),
+      order_by: u.id
+    )
+    |> Repo.all()
+  end
 
-    Repo.all(query)
+  def count_all_except_one(user) do
+    query =
+      from(
+        u in User,
+        where: u.id != ^user.id
+      )
+
+    Repo.aggregate(query, :count, :id)
   end
 
   defp do_search(subquery, for_user, options \\ []) do
diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex
index ef72509fe..d75b7e7e7 100644
--- a/lib/pleroma/web/admin_api/admin_api_controller.ex
+++ b/lib/pleroma/web/admin_api/admin_api_controller.ex
@@ -3,6 +3,8 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.AdminAPI.AdminAPIController do
+  @users_page_size 50
+
   use Pleroma.Web, :controller
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.Relay
@@ -61,11 +63,19 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
          do: json_response(conn, :no_content, "")
   end
 
-  def list_users(%{assigns: %{user: admin}} = conn, _data) do
-    users = User.all_except_one(admin)
-
-    conn
-    |> json(UserView.render("index_for_admin.json", %{users: users}))
+  def list_users(%{assigns: %{user: admin}} = conn, %{"page" => page_string}) do
+    with {page, _} <- Integer.parse(page_string),
+         users <- User.all_except_one(admin, page, @users_page_size),
+         count <- User.count_all_except_one(admin),
+         do:
+           conn
+           |> json(
+             UserView.render("index_for_admin.json", %{
+               users: users,
+               count: count,
+               page_size: @users_page_size
+             })
+           )
   end
 
   def right_add(conn, %{"permission_group" => permission_group, "nickname" => nickname})
diff --git a/lib/pleroma/web/twitter_api/views/user_view.ex b/lib/pleroma/web/twitter_api/views/user_view.ex
index c5034cf36..e8514d3ba 100644
--- a/lib/pleroma/web/twitter_api/views/user_view.ex
+++ b/lib/pleroma/web/twitter_api/views/user_view.ex
@@ -27,9 +27,12 @@ defmodule Pleroma.Web.TwitterAPI.UserView do
       else: %{}
   end
 
-  def render("index_for_admin.json", %{users: users} = opts) do
-    users
-    |> render_many(UserView, "show_for_admin.json", opts)
+  def render("index_for_admin.json", %{users: users, count: count, page_size: page_size} = opts) do
+    %{
+      users: render_many(users, UserView, "show_for_admin.json", opts),
+      count: count,
+      page_size: page_size
+    }
   end
 
   def render("show_for_admin.json", %{user: user}) do
diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs
index f6ae16844..1b0a2f5be 100644
--- a/test/web/admin_api/admin_api_controller_test.exs
+++ b/test/web/admin_api/admin_api_controller_test.exs
@@ -338,15 +338,19 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
     conn =
       build_conn()
       |> assign(:user, admin)
-      |> get("/api/pleroma/admin/users")
+      |> get("/api/pleroma/admin/users?page=1")
 
-    assert json_response(conn, 200) == [
-             %{
-               "deactivated" => user.info.deactivated,
-               "id" => user.id,
-               "nickname" => user.nickname
-             }
-           ]
+    assert json_response(conn, 200) == %{
+             "count" => 1,
+             "page_size" => 50,
+             "users" => [
+               %{
+                 "deactivated" => user.info.deactivated,
+                 "id" => user.id,
+                 "nickname" => user.nickname
+               }
+             ]
+           }
   end
 
   test "PATCH /api/pleroma/admin/users/:nickname/toggle_activation" do

From 6b11011039dca5090c3a7b7b2a01f32b666be380 Mon Sep 17 00:00:00 2001
From: eugenijm <eugenijm@protonmail.com>
Date: Thu, 28 Feb 2019 08:31:33 +0300
Subject: [PATCH 26/68] Added deactivated to the user view

---
 lib/pleroma/web/twitter_api/views/user_view.ex | 7 +++++++
 test/web/twitter_api/views/user_view_test.exs  | 7 +++++++
 2 files changed, 14 insertions(+)

diff --git a/lib/pleroma/web/twitter_api/views/user_view.ex b/lib/pleroma/web/twitter_api/views/user_view.ex
index c5034cf36..22f33e0b5 100644
--- a/lib/pleroma/web/twitter_api/views/user_view.ex
+++ b/lib/pleroma/web/twitter_api/views/user_view.ex
@@ -132,6 +132,7 @@ defmodule Pleroma.Web.TwitterAPI.UserView do
           "confirmation_pending" => user_info.confirmation_pending,
           "tags" => user.tags
         }
+        |> maybe_with_activation_status(user, for_user)
         |> maybe_with_follow_request_count(user, for_user)
     }
 
@@ -148,6 +149,12 @@ defmodule Pleroma.Web.TwitterAPI.UserView do
     end
   end
 
+  defp maybe_with_activation_status(data, user, %User{info: %{is_admin: true}}) do
+    Map.put(data, "deactivated", user.info.deactivated)
+  end
+
+  defp maybe_with_activation_status(data, _, _), do: data
+
   defp maybe_with_follow_request_count(data, %User{id: id, info: %{locked: true}} = user, %User{
          id: id
        }) do
diff --git a/test/web/twitter_api/views/user_view_test.exs b/test/web/twitter_api/views/user_view_test.exs
index 95e52ca46..114f24a1c 100644
--- a/test/web/twitter_api/views/user_view_test.exs
+++ b/test/web/twitter_api/views/user_view_test.exs
@@ -239,6 +239,13 @@ defmodule Pleroma.Web.TwitterAPI.UserViewTest do
     assert represented["role"] == nil
   end
 
+  test "A regular user for the admin", %{user: user} do
+    admin = insert(:user, %{info: %{is_admin: true}})
+    represented = UserView.render("show.json", %{user: user, for: admin})
+
+    assert represented["pleroma"]["deactivated"] == false
+  end
+
   test "A blocked user for the blocker" do
     user = insert(:user)
     blocker = insert(:user)

From b6f915313f59223002a0eff88c1eefb00ca5c8f3 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Thu, 28 Feb 2019 13:00:54 +0300
Subject: [PATCH 27/68] Made auth customization be runtime-configurable.

---
 lib/pleroma/web/auth/database_authenticator.ex | 14 ++++++++------
 lib/pleroma/web/oauth/oauth_controller.ex      |  2 +-
 lib/pleroma/web/web.ex                         |  6 +++++-
 3 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/lib/pleroma/web/auth/database_authenticator.ex b/lib/pleroma/web/auth/database_authenticator.ex
index 02a16b634..e78068b03 100644
--- a/lib/pleroma/web/auth/database_authenticator.ex
+++ b/lib/pleroma/web/auth/database_authenticator.ex
@@ -5,14 +5,16 @@
 defmodule Pleroma.Web.Auth.DatabaseAuthenticator do
   alias Pleroma.User
 
-  @implementation Pleroma.Config.get(
-                    Pleroma.Web.Auth.DatabaseAuthenticator,
-                    Pleroma.Web.Auth.PleromaDatabaseAuthenticator
-                  )
+  def implementation do
+    Pleroma.Config.get(
+      Pleroma.Web.Auth.DatabaseAuthenticator,
+      Pleroma.Web.Auth.PleromaDatabaseAuthenticator
+    )
+  end
 
   @callback get_user(Plug.Conn.t()) :: {:ok, User.t()} | {:error, any()}
-  defdelegate get_user(plug), to: @implementation
+  def get_user(plug), do: implementation().get_user(plug)
 
   @callback handle_error(Plug.Conn.t(), any()) :: any()
-  defdelegate handle_error(plug, error), to: @implementation
+  def handle_error(plug, error), do: implementation().handle_error(plug, error)
 end
diff --git a/lib/pleroma/web/oauth/oauth_controller.ex b/lib/pleroma/web/oauth/oauth_controller.ex
index 02c0babd2..5c2b0507c 100644
--- a/lib/pleroma/web/oauth/oauth_controller.ex
+++ b/lib/pleroma/web/oauth/oauth_controller.ex
@@ -25,7 +25,7 @@ defmodule Pleroma.Web.OAuth.OAuthController do
     available_scopes = (app && app.scopes) || []
     scopes = oauth_scopes(params, nil) || available_scopes
 
-    template = Application.get_env(:pleroma, :auth_template, "show.html")
+    template = Pleroma.Config.get(:auth_template, "show.html")
 
     render(conn, template, %{
       response_type: params["response_type"],
diff --git a/lib/pleroma/web/web.ex b/lib/pleroma/web/web.ex
index 4bf07a6ef..66813e4dd 100644
--- a/lib/pleroma/web/web.ex
+++ b/lib/pleroma/web/web.ex
@@ -27,7 +27,11 @@ defmodule Pleroma.Web do
       import Pleroma.Web.Gettext
       import Pleroma.Web.Router.Helpers
 
-      plug(:put_layout, Application.get_env(:pleroma, :app_layout, "app.html"))
+      plug(:set_put_layout)
+
+      defp set_put_layout(conn, _) do
+        put_layout(conn, Pleroma.Config.get(:app_layout, "app.html"))
+      end
     end
   end
 

From 4e77f68414c1dd6a906d1d9b6b78916db5c213d5 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Thu, 28 Feb 2019 13:58:58 +0300
Subject: [PATCH 28/68] Added `auth_template/0` to DatabaseAuthenticator.

---
 lib/pleroma/web/auth/database_authenticator.ex         | 5 +++++
 lib/pleroma/web/auth/pleroma_database_authenticator.ex | 2 ++
 lib/pleroma/web/oauth/oauth_controller.ex              | 4 +---
 3 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/web/auth/database_authenticator.ex b/lib/pleroma/web/auth/database_authenticator.ex
index e78068b03..494f28f23 100644
--- a/lib/pleroma/web/auth/database_authenticator.ex
+++ b/lib/pleroma/web/auth/database_authenticator.ex
@@ -17,4 +17,9 @@ defmodule Pleroma.Web.Auth.DatabaseAuthenticator do
 
   @callback handle_error(Plug.Conn.t(), any()) :: any()
   def handle_error(plug, error), do: implementation().handle_error(plug, error)
+
+  @callback auth_template() :: String.t() | nil
+  def auth_template do
+    implementation().auth_template() || Pleroma.Config.get(:auth_template, "show.html")
+  end
 end
diff --git a/lib/pleroma/web/auth/pleroma_database_authenticator.ex b/lib/pleroma/web/auth/pleroma_database_authenticator.ex
index 39aa1a586..0780388bc 100644
--- a/lib/pleroma/web/auth/pleroma_database_authenticator.ex
+++ b/lib/pleroma/web/auth/pleroma_database_authenticator.ex
@@ -23,4 +23,6 @@ defmodule Pleroma.Web.Auth.PleromaDatabaseAuthenticator do
   def handle_error(%Plug.Conn{} = _conn, error) do
     error
   end
+
+  def auth_template, do: nil
 end
diff --git a/lib/pleroma/web/oauth/oauth_controller.ex b/lib/pleroma/web/oauth/oauth_controller.ex
index 5c2b0507c..99346f399 100644
--- a/lib/pleroma/web/oauth/oauth_controller.ex
+++ b/lib/pleroma/web/oauth/oauth_controller.ex
@@ -25,9 +25,7 @@ defmodule Pleroma.Web.OAuth.OAuthController do
     available_scopes = (app && app.scopes) || []
     scopes = oauth_scopes(params, nil) || available_scopes
 
-    template = Pleroma.Config.get(:auth_template, "show.html")
-
-    render(conn, template, %{
+    render(conn, DatabaseAuthenticator.auth_template(), %{
       response_type: params["response_type"],
       client_id: params["client_id"],
       available_scopes: available_scopes,

From 3281a3f074f8aac6a05a24cc2eee4e5bad2275bd Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Thu, 28 Feb 2019 14:12:41 +0300
Subject: [PATCH 29/68] Renamed *DatabaseAuthenticator to *Authenticator.

---
 .../auth/{database_authenticator.ex => authenticator.ex}  | 6 +++---
 ...database_authenticator.ex => pleroma_authenticator.ex} | 4 ++--
 lib/pleroma/web/oauth/oauth_controller.ex                 | 8 ++++----
 3 files changed, 9 insertions(+), 9 deletions(-)
 rename lib/pleroma/web/auth/{database_authenticator.ex => authenticator.ex} (81%)
 rename lib/pleroma/web/auth/{pleroma_database_authenticator.ex => pleroma_authenticator.ex} (85%)

diff --git a/lib/pleroma/web/auth/database_authenticator.ex b/lib/pleroma/web/auth/authenticator.ex
similarity index 81%
rename from lib/pleroma/web/auth/database_authenticator.ex
rename to lib/pleroma/web/auth/authenticator.ex
index 494f28f23..82267c595 100644
--- a/lib/pleroma/web/auth/database_authenticator.ex
+++ b/lib/pleroma/web/auth/authenticator.ex
@@ -2,13 +2,13 @@
 # Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.Web.Auth.DatabaseAuthenticator do
+defmodule Pleroma.Web.Auth.Authenticator do
   alias Pleroma.User
 
   def implementation do
     Pleroma.Config.get(
-      Pleroma.Web.Auth.DatabaseAuthenticator,
-      Pleroma.Web.Auth.PleromaDatabaseAuthenticator
+      Pleroma.Web.Auth.Authenticator,
+      Pleroma.Web.Auth.PleromaAuthenticator
     )
   end
 
diff --git a/lib/pleroma/web/auth/pleroma_database_authenticator.ex b/lib/pleroma/web/auth/pleroma_authenticator.ex
similarity index 85%
rename from lib/pleroma/web/auth/pleroma_database_authenticator.ex
rename to lib/pleroma/web/auth/pleroma_authenticator.ex
index 0780388bc..3cc19af01 100644
--- a/lib/pleroma/web/auth/pleroma_database_authenticator.ex
+++ b/lib/pleroma/web/auth/pleroma_authenticator.ex
@@ -2,11 +2,11 @@
 # Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.Web.Auth.PleromaDatabaseAuthenticator do
+defmodule Pleroma.Web.Auth.PleromaAuthenticator do
   alias Pleroma.User
   alias Comeonin.Pbkdf2
 
-  @behaviour Pleroma.Web.Auth.DatabaseAuthenticator
+  @behaviour Pleroma.Web.Auth.Authenticator
 
   def get_user(%Plug.Conn{} = conn) do
     %{"authorization" => %{"name" => name, "password" => password}} = conn.params
diff --git a/lib/pleroma/web/oauth/oauth_controller.ex b/lib/pleroma/web/oauth/oauth_controller.ex
index 99346f399..b16e3b2a7 100644
--- a/lib/pleroma/web/oauth/oauth_controller.ex
+++ b/lib/pleroma/web/oauth/oauth_controller.ex
@@ -5,7 +5,7 @@
 defmodule Pleroma.Web.OAuth.OAuthController do
   use Pleroma.Web, :controller
 
-  alias Pleroma.Web.Auth.DatabaseAuthenticator
+  alias Pleroma.Web.Auth.Authenticator
   alias Pleroma.Web.OAuth.Authorization
   alias Pleroma.Web.OAuth.Token
   alias Pleroma.Web.OAuth.App
@@ -25,7 +25,7 @@ defmodule Pleroma.Web.OAuth.OAuthController do
     available_scopes = (app && app.scopes) || []
     scopes = oauth_scopes(params, nil) || available_scopes
 
-    render(conn, DatabaseAuthenticator.auth_template(), %{
+    render(conn, Authenticator.auth_template(), %{
       response_type: params["response_type"],
       client_id: params["client_id"],
       available_scopes: available_scopes,
@@ -43,7 +43,7 @@ defmodule Pleroma.Web.OAuth.OAuthController do
             "redirect_uri" => redirect_uri
           } = auth_params
       }) do
-    with {_, {:ok, %User{} = user}} <- {:get_user, DatabaseAuthenticator.get_user(conn)},
+    with {_, {:ok, %User{} = user}} <- {:get_user, Authenticator.get_user(conn)},
          %App{} = app <- Repo.get_by(App, client_id: client_id),
          true <- redirect_uri in String.split(app.redirect_uris),
          scopes <- oauth_scopes(auth_params, []),
@@ -96,7 +96,7 @@ defmodule Pleroma.Web.OAuth.OAuthController do
         |> authorize(auth_params)
 
       error ->
-        DatabaseAuthenticator.handle_error(conn, error)
+        Authenticator.handle_error(conn, error)
     end
   end
 

From 72b7a0797ec42a021f4f8f50dce859fb0f12bf75 Mon Sep 17 00:00:00 2001
From: Maxim Filippov <colixer@gmail.com>
Date: Thu, 28 Feb 2019 17:43:09 +0300
Subject: [PATCH 30/68] Use Mastodon API views in Admin API

---
 .../web/admin_api/admin_api_controller.ex     |  6 ++---
 .../mastodon_api/views/admin/account_view.ex  | 25 +++++++++++++++++++
 .../web/twitter_api/views/user_view.ex        | 17 -------------
 3 files changed, 28 insertions(+), 20 deletions(-)
 create mode 100644 lib/pleroma/web/mastodon_api/views/admin/account_view.ex

diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex
index d75b7e7e7..d8e3d57e1 100644
--- a/lib/pleroma/web/admin_api/admin_api_controller.ex
+++ b/lib/pleroma/web/admin_api/admin_api_controller.ex
@@ -8,7 +8,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
   use Pleroma.Web, :controller
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.Relay
-  alias Pleroma.Web.TwitterAPI.UserView
+  alias Pleroma.Web.MastodonAPI.Admin.AccountView
 
   import Pleroma.Web.ControllerHelper, only: [json_response: 3]
 
@@ -50,7 +50,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     {:ok, updated_user} = User.deactivate(user, !user.info.deactivated)
 
     conn
-    |> json(UserView.render("show_for_admin.json", %{user: updated_user}))
+    |> json(AccountView.render("show.json", %{user: updated_user}))
   end
 
   def tag_users(conn, %{"nicknames" => nicknames, "tags" => tags}) do
@@ -70,7 +70,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
          do:
            conn
            |> json(
-             UserView.render("index_for_admin.json", %{
+             AccountView.render("index.json", %{
                users: users,
                count: count,
                page_size: @users_page_size
diff --git a/lib/pleroma/web/mastodon_api/views/admin/account_view.ex b/lib/pleroma/web/mastodon_api/views/admin/account_view.ex
new file mode 100644
index 000000000..74ca13564
--- /dev/null
+++ b/lib/pleroma/web/mastodon_api/views/admin/account_view.ex
@@ -0,0 +1,25 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.MastodonAPI.Admin.AccountView do
+  use Pleroma.Web, :view
+
+  alias Pleroma.Web.MastodonAPI.Admin.AccountView
+
+  def render("index.json", %{users: users, count: count, page_size: page_size}) do
+    %{
+      users: render_many(users, AccountView, "show.json", as: :user),
+      count: count,
+      page_size: page_size
+    }
+  end
+
+  def render("show.json", %{user: user}) do
+    %{
+      "id" => user.id,
+      "nickname" => user.nickname,
+      "deactivated" => user.info.deactivated
+    }
+  end
+end
diff --git a/lib/pleroma/web/twitter_api/views/user_view.ex b/lib/pleroma/web/twitter_api/views/user_view.ex
index e8514d3ba..df7384476 100644
--- a/lib/pleroma/web/twitter_api/views/user_view.ex
+++ b/lib/pleroma/web/twitter_api/views/user_view.ex
@@ -9,7 +9,6 @@ defmodule Pleroma.Web.TwitterAPI.UserView do
   alias Pleroma.User
   alias Pleroma.Web.CommonAPI.Utils
   alias Pleroma.Web.MediaProxy
-  alias Pleroma.Web.TwitterAPI.UserView
 
   def render("show.json", %{user: user = %User{}} = assigns) do
     render_one(user, Pleroma.Web.TwitterAPI.UserView, "user.json", assigns)
@@ -27,22 +26,6 @@ defmodule Pleroma.Web.TwitterAPI.UserView do
       else: %{}
   end
 
-  def render("index_for_admin.json", %{users: users, count: count, page_size: page_size} = opts) do
-    %{
-      users: render_many(users, UserView, "show_for_admin.json", opts),
-      count: count,
-      page_size: page_size
-    }
-  end
-
-  def render("show_for_admin.json", %{user: user}) do
-    %{
-      "id" => user.id,
-      "nickname" => user.nickname,
-      "deactivated" => user.info.deactivated
-    }
-  end
-
   def render("short.json", %{
         user: %User{
           nickname: nickname,

From 70e82a3465ee10004d0ae347934524e779bd778a Mon Sep 17 00:00:00 2001
From: Maxim Filippov <colixer@gmail.com>
Date: Thu, 28 Feb 2019 17:54:02 +0300
Subject: [PATCH 31/68] Add test for the second page

---
 .../admin_api/admin_api_controller_test.exs   | 54 ++++++++++++-------
 1 file changed, 36 insertions(+), 18 deletions(-)

diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs
index 1b0a2f5be..893387ef5 100644
--- a/test/web/admin_api/admin_api_controller_test.exs
+++ b/test/web/admin_api/admin_api_controller_test.exs
@@ -331,26 +331,44 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
     assert conn.status == 200
   end
 
-  test "GET /api/pleroma/admin/users" do
-    admin = insert(:user, info: %{is_admin: true})
-    user = insert(:user)
+  describe "GET /api/pleroma/admin/users" do
+    test "renders users array for the first page" do
+      admin = insert(:user, info: %{is_admin: true})
+      user = insert(:user)
 
-    conn =
-      build_conn()
-      |> assign(:user, admin)
-      |> get("/api/pleroma/admin/users?page=1")
+      conn =
+        build_conn()
+        |> assign(:user, admin)
+        |> get("/api/pleroma/admin/users?page=1")
 
-    assert json_response(conn, 200) == %{
-             "count" => 1,
-             "page_size" => 50,
-             "users" => [
-               %{
-                 "deactivated" => user.info.deactivated,
-                 "id" => user.id,
-                 "nickname" => user.nickname
-               }
-             ]
-           }
+      assert json_response(conn, 200) == %{
+               "count" => 1,
+               "page_size" => 50,
+               "users" => [
+                 %{
+                   "deactivated" => user.info.deactivated,
+                   "id" => user.id,
+                   "nickname" => user.nickname
+                 }
+               ]
+             }
+    end
+
+    test "renders empty array for the second page" do
+      admin = insert(:user, info: %{is_admin: true})
+      user = insert(:user)
+
+      conn =
+        build_conn()
+        |> assign(:user, admin)
+        |> get("/api/pleroma/admin/users?page=2")
+
+      assert json_response(conn, 200) == %{
+               "count" => 1,
+               "page_size" => 50,
+               "users" => []
+             }
+    end
   end
 
   test "PATCH /api/pleroma/admin/users/:nickname/toggle_activation" do

From e10ca35b3de0ca1b32ab5fd57f559a1cec1e5c3e Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Thu, 28 Feb 2019 14:53:54 +0000
Subject: [PATCH 32/68] mix: update dependencies for cowboy 2.0

---
 mix.exs  |  5 ++---
 mix.lock | 10 +++++-----
 2 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/mix.exs b/mix.exs
index 5392d94d1..0fe9c6ec3 100644
--- a/mix.exs
+++ b/mix.exs
@@ -55,9 +55,8 @@ defmodule Pleroma.Mixfile do
   # Type `mix help deps` for examples and options.
   defp deps do
     [
-      # Until Phoenix 1.4.1 is released
-      {:phoenix, github: "phoenixframework/phoenix", branch: "v1.4"},
-      {:plug_cowboy, "~> 1.0"},
+      {:phoenix, "~> 1.4.1"},
+      {:plug_cowboy, "~> 2.0"},
       {:phoenix_pubsub, "~> 1.1"},
       {:phoenix_ecto, "~> 3.3"},
       {:postgrex, ">= 0.13.5"},
diff --git a/mix.lock b/mix.lock
index 918702444..50e742fbd 100644
--- a/mix.lock
+++ b/mix.lock
@@ -9,8 +9,8 @@
   "comeonin": {:hex, :comeonin, "4.1.1", "c7304fc29b45b897b34142a91122bc72757bc0c295e9e824999d5179ffc08416", [:mix], [{:argon2_elixir, "~> 1.2", [hex: :argon2_elixir, repo: "hexpm", optional: true]}, {:bcrypt_elixir, "~> 0.12.1 or ~> 1.0", [hex: :bcrypt_elixir, repo: "hexpm", optional: true]}, {:pbkdf2_elixir, "~> 0.12", [hex: :pbkdf2_elixir, repo: "hexpm", optional: true]}], "hexpm"},
   "connection": {:hex, :connection, "1.0.4", "a1cae72211f0eef17705aaededacac3eb30e6625b04a6117c1b2db6ace7d5976", [:mix], [], "hexpm"},
   "cors_plug": {:hex, :cors_plug, "1.5.2", "72df63c87e4f94112f458ce9d25800900cc88608c1078f0e4faddf20933eda6e", [:mix], [{:plug, "~> 1.3 or ~> 1.4 or ~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"},
-  "cowboy": {:hex, :cowboy, "1.1.2", "61ac29ea970389a88eca5a65601460162d370a70018afe6f949a29dca91f3bb0", [:rebar3], [{:cowlib, "~> 1.0.2", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "~> 1.3.2", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm"},
-  "cowlib": {:hex, :cowlib, "1.0.2", "9d769a1d062c9c3ac753096f868ca121e2730b9a377de23dec0f7e08b1df84ee", [:make], [], "hexpm"},
+  "cowboy": {:hex, :cowboy, "2.6.1", "f2e06f757c337b3b311f9437e6e072b678fcd71545a7b2865bdaa154d078593f", [:rebar3], [{:cowlib, "~> 2.7.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "~> 1.7.1", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm"},
+  "cowlib": {:hex, :cowlib, "2.7.0", "3ef16e77562f9855a2605900cedb15c1462d76fb1be6a32fc3ae91973ee543d2", [:rebar3], [], "hexpm"},
   "credo": {:hex, :credo, "0.9.3", "76fa3e9e497ab282e0cf64b98a624aa11da702854c52c82db1bf24e54ab7c97a", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:poison, ">= 0.0.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm"},
   "crypt": {:git, "https://github.com/msantos/crypt", "1f2b58927ab57e72910191a7ebaeff984382a1d3", [ref: "1f2b58927ab57e72910191a7ebaeff984382a1d3"]},
   "db_connection": {:hex, :db_connection, "1.1.3", "89b30ca1ef0a3b469b1c779579590688561d586694a3ce8792985d4d7e575a61", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}, {:poolboy, "~> 1.5", [hex: :poolboy, repo: "hexpm", optional: true]}, {:sbroker, "~> 1.0", [hex: :sbroker, repo: "hexpm", optional: true]}], "hexpm"},
@@ -45,17 +45,17 @@
   "nimble_parsec": {:hex, :nimble_parsec, "0.4.0", "ee261bb53214943679422be70f1658fff573c5d0b0a1ecd0f18738944f818efe", [:mix], [], "hexpm"},
   "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm"},
   "pbkdf2_elixir": {:hex, :pbkdf2_elixir, "0.12.3", "6706a148809a29c306062862c803406e88f048277f6e85b68faf73291e820b84", [:mix], [], "hexpm"},
-  "phoenix": {:git, "https://github.com/phoenixframework/phoenix.git", "ea22dc50b574178a300ecd19253443960407df93", [branch: "v1.4"]},
+  "phoenix": {:hex, :phoenix, "1.4.1", "801f9d632808657f1f7c657c8bbe624caaf2ba91429123ebe3801598aea4c3d9", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"},
   "phoenix_ecto": {:hex, :phoenix_ecto, "3.3.0", "702f6e164512853d29f9d20763493f2b3bcfcb44f118af2bc37bb95d0801b480", [:mix], [{:ecto, "~> 2.1", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.9", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"},
   "phoenix_html": {:hex, :phoenix_html, "2.13.1", "fa8f034b5328e2dfa0e4131b5569379003f34bc1fafdaa84985b0b9d2f12e68b", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"},
   "phoenix_pubsub": {:hex, :phoenix_pubsub, "1.1.1", "6668d787e602981f24f17a5fbb69cc98f8ab085114ebfac6cc36e10a90c8e93c", [:mix], [], "hexpm"},
   "plug": {:hex, :plug, "1.7.2", "d7b7db7fbd755e8283b6c0a50be71ec0a3d67d9213d74422d9372effc8e87fd1", [:mix], [{:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}], "hexpm"},
-  "plug_cowboy": {:hex, :plug_cowboy, "1.0.0", "2e2a7d3409746d335f451218b8bb0858301c3de6d668c3052716c909936eb57a", [:mix], [{:cowboy, "~> 1.0", [hex: :cowboy, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"},
+  "plug_cowboy": {:hex, :plug_cowboy, "2.0.1", "d798f8ee5acc86b7d42dbe4450b8b0dadf665ce588236eb0a751a132417a980e", [:mix], [{:cowboy, "~> 2.5", [hex: :cowboy, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"},
   "plug_crypto": {:hex, :plug_crypto, "1.0.0", "18e49317d3fa343f24620ed22795ec29d4a5e602d52d1513ccea0b07d8ea7d4d", [:mix], [], "hexpm"},
   "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm"},
   "poolboy": {:hex, :poolboy, "1.5.1", "6b46163901cfd0a1b43d692657ed9d7e599853b3b21b95ae5ae0a777cf9b6ca8", [:rebar], [], "hexpm"},
   "postgrex": {:hex, :postgrex, "0.13.5", "3d931aba29363e1443da167a4b12f06dcd171103c424de15e5f3fc2ba3e6d9c5", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 1.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: false]}], "hexpm"},
-  "ranch": {:hex, :ranch, "1.3.2", "e4965a144dc9fbe70e5c077c65e73c57165416a901bd02ea899cfd95aa890986", [:rebar3], [], "hexpm"},
+  "ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm"},
   "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm"},
   "swoosh": {:hex, :swoosh, "0.20.0", "9a6c13822c9815993c03b6f8fccc370fcffb3c158d9754f67b1fdee6b3a5d928", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.12", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug, "~> 1.4", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm"},
   "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]},

From bc53dff5b6c7814a1cfeca43fc2a843e1e306501 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Thu, 28 Feb 2019 15:17:01 +0000
Subject: [PATCH 33/68] mastodon api: websocket: update code for cowboy 2.x

---
 .../web/mastodon_api/websocket_handler.ex     | 39 +++++++++----------
 1 file changed, 18 insertions(+), 21 deletions(-)

diff --git a/lib/pleroma/web/mastodon_api/websocket_handler.ex b/lib/pleroma/web/mastodon_api/websocket_handler.ex
index ea75070c4..f19f26b9a 100644
--- a/lib/pleroma/web/mastodon_api/websocket_handler.ex
+++ b/lib/pleroma/web/mastodon_api/websocket_handler.ex
@@ -9,7 +9,7 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do
   alias Pleroma.Repo
   alias Pleroma.User
 
-  @behaviour :cowboy_websocket_handler
+  @behaviour :cowboy_websocket
 
   @streams [
     "public",
@@ -23,40 +23,37 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do
   ]
   @anonymous_streams ["public", "public:local", "hashtag"]
 
-  # Handled by periodic keepalive in Pleroma.Web.Streamer.
-  @timeout :infinity
-
-  def init(_type, _req, _opts) do
-    {:upgrade, :protocol, :cowboy_websocket}
-  end
-
-  def websocket_init(_type, req, _opts) do
+  def init(req, _state) do
     with {qs, req} <- :cowboy_req.qs(req),
          params <- :cow_qs.parse_qs(qs),
          access_token <- List.keyfind(params, "access_token", 0),
          {_, stream} <- List.keyfind(params, "stream", 0),
          {:ok, user} <- allow_request(stream, access_token),
          topic when is_binary(topic) <- expand_topic(stream, params) do
-      send(self(), :subscribe)
-      {:ok, req, %{user: user, topic: topic}, @timeout}
+      {:cowboy_websocket, req, %{user: user, topic: topic}}
     else
       {:error, code} ->
         Logger.debug("#{__MODULE__} denied connection: #{inspect(code)} - #{inspect(req)}")
         {:ok, req} = :cowboy_req.reply(code, req)
-        {:shutdown, req}
+        {:stop, req}
 
       error ->
         Logger.debug("#{__MODULE__} denied connection: #{inspect(error)} - #{inspect(req)}")
-        {:shutdown, req}
+        {:stop, req}
     end
   end
 
-  # We never receive messages.
-  def websocket_handle(_frame, req, state) do
-    {:ok, req, state}
+  def websocket_init(state) do
+    send(self(), :subscribe)
+    {:ok, state}
   end
 
-  def websocket_info(:subscribe, req, state) do
+  # We never receive messages.
+  def websocket_handle(_frame, state) do
+    {:ok, state}
+  end
+
+  def websocket_info(:subscribe, state) do
     Logger.debug(
       "#{__MODULE__} accepted websocket connection for user #{
         (state.user || %{id: "anonymous"}).id
@@ -64,14 +61,14 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do
     )
 
     Pleroma.Web.Streamer.add_socket(state.topic, streamer_socket(state))
-    {:ok, req, state}
+    {:ok, state}
   end
 
-  def websocket_info({:text, message}, req, state) do
-    {:reply, {:text, message}, req, state}
+  def websocket_info({:text, message}, state) do
+    {:reply, {:text, message}, state}
   end
 
-  def websocket_terminate(reason, _req, state) do
+  def terminate(reason, _req, state) do
     Logger.debug(
       "#{__MODULE__} terminating websocket connection for user #{
         (state.user || %{id: "anonymous"}).id

From 9aec00d71101b848aab6b61ad6c5335284fb339b Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Thu, 28 Feb 2019 15:43:38 +0000
Subject: [PATCH 34/68] config: update config for cowboy 2 endpoints

---
 config/config.exs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/config/config.exs b/config/config.exs
index 7e4ac1100..374b4d86b 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -93,10 +93,10 @@ config :pleroma, Pleroma.Web.Endpoint,
     dispatch: [
       {:_,
        [
-         {"/api/v1/streaming", Elixir.Pleroma.Web.MastodonAPI.WebsocketHandler, []},
-         {"/socket/websocket", Phoenix.Endpoint.CowboyWebSocket,
+         {"/api/v1/streaming", Pleroma.Web.MastodonAPI.WebsocketHandler, []},
+         {"/socket/websocket", Phoenix.Endpoint.CowboyWebsocket,
           {nil, {Pleroma.Web.Endpoint, Pleroma.Web.UserSocket, websocket_config}}},
-         {:_, Plug.Adapters.Cowboy.Handler, {Pleroma.Web.Endpoint, []}}
+         {:_, Phoenix.Endpoint.Cowboy2Handler, {Pleroma.Web.Endpoint, []}}
        ]}
     ]
   ],

From f1d37a5e2399d06c8a2178c98825e77e5b381460 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Thu, 28 Feb 2019 16:02:48 +0000
Subject: [PATCH 35/68] mastodon websocket: use pattern match to get query
 data, robustly handle errors

---
 lib/pleroma/web/mastodon_api/websocket_handler.ex | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/web/mastodon_api/websocket_handler.ex b/lib/pleroma/web/mastodon_api/websocket_handler.ex
index f19f26b9a..edab961f0 100644
--- a/lib/pleroma/web/mastodon_api/websocket_handler.ex
+++ b/lib/pleroma/web/mastodon_api/websocket_handler.ex
@@ -23,9 +23,8 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do
   ]
   @anonymous_streams ["public", "public:local", "hashtag"]
 
-  def init(req, _state) do
-    with {qs, req} <- :cowboy_req.qs(req),
-         params <- :cow_qs.parse_qs(qs),
+  def init(%{qs: qs} = req, _state) do
+    with params <- :cow_qs.parse_qs(qs),
          access_token <- List.keyfind(params, "access_token", 0),
          {_, stream} <- List.keyfind(params, "stream", 0),
          {:ok, user} <- allow_request(stream, access_token),
@@ -39,6 +38,7 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do
 
       error ->
         Logger.debug("#{__MODULE__} denied connection: #{inspect(error)} - #{inspect(req)}")
+        {:ok, req} = :cowboy_req.reply(400, req)
         {:stop, req}
     end
   end

From 46f29b9da1cfdcc2ab14616f999f061fa0c87ddc Mon Sep 17 00:00:00 2001
From: Maxim Filippov <colixer@gmail.com>
Date: Thu, 28 Feb 2019 19:04:47 +0300
Subject: [PATCH 36/68] Add search users endpoint

---
 lib/pleroma/user.ex                           |  8 +++----
 .../web/admin_api/admin_api_controller.ex     | 13 ++++++++++
 lib/pleroma/web/router.ex                     |  1 +
 .../admin_api/admin_api_controller_test.exs   | 24 ++++++++++++++++++-
 4 files changed, 41 insertions(+), 5 deletions(-)

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 80e4ae296..52df171c5 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -755,18 +755,18 @@ defmodule Pleroma.User do
     Repo.all(query)
   end
 
-  def search(query, resolve \\ false, for_user \\ nil) do
+  def search(query, resolve \\ false, for_user \\ nil, limit \\ 20) do
     # Strip the beginning @ off if there is a query
     query = String.trim_leading(query, "@")
 
     if resolve, do: get_or_fetch(query)
 
-    fts_results = do_search(fts_search_subquery(query), for_user)
+    fts_results = do_search(fts_search_subquery(query), for_user, %{limit: limit})
 
     {:ok, trigram_results} =
       Repo.transaction(fn ->
         Ecto.Adapters.SQL.query(Repo, "select set_limit(0.25)", [])
-        do_search(trigram_search_subquery(query), for_user)
+        do_search(trigram_search_subquery(query), for_user, %{limit: limit})
       end)
 
     Enum.uniq_by(fts_results ++ trigram_results, & &1.id)
@@ -793,7 +793,7 @@ defmodule Pleroma.User do
     Repo.aggregate(query, :count, :id)
   end
 
-  defp do_search(subquery, for_user, options \\ []) do
+  defp do_search(subquery, for_user, options) do
     q =
       from(
         s in subquery(subquery),
diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex
index d8e3d57e1..37159cd40 100644
--- a/lib/pleroma/web/admin_api/admin_api_controller.ex
+++ b/lib/pleroma/web/admin_api/admin_api_controller.ex
@@ -78,6 +78,19 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
            )
   end
 
+  def search_users(%{assigns: %{user: admin}} = conn, %{"query" => query}) do
+    users = User.search(query, true, admin, @users_page_size)
+
+    conn
+    |> json(
+      AccountView.render("index.json", %{
+        users: users,
+        count: length(users),
+        page_size: @users_page_size
+      })
+    )
+  end
+
   def right_add(conn, %{"permission_group" => permission_group, "nickname" => nickname})
       when permission_group in ["moderator", "admin"] do
     user = User.get_by_nickname(nickname)
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index 3b1fd46a5..6fcb46878 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -140,6 +140,7 @@ defmodule Pleroma.Web.Router do
     pipe_through([:admin_api, :oauth_write])
 
     get("/users", AdminAPIController, :list_users)
+    get("/users/search", AdminAPIController, :search_users)
     delete("/user", AdminAPIController, :user_delete)
     patch("/users/:nickname/toggle_activation", AdminAPIController, :user_toggle_activation)
     post("/user", AdminAPIController, :user_create)
diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs
index 893387ef5..14625af32 100644
--- a/test/web/admin_api/admin_api_controller_test.exs
+++ b/test/web/admin_api/admin_api_controller_test.exs
@@ -356,7 +356,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
 
     test "renders empty array for the second page" do
       admin = insert(:user, info: %{is_admin: true})
-      user = insert(:user)
+      insert(:user)
 
       conn =
         build_conn()
@@ -387,4 +387,26 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
                "nickname" => user.nickname
              }
   end
+
+  test "GET /api/pleroma/admin/users/search" do
+    admin = insert(:user, info: %{is_admin: true})
+    user = insert(:user, nickname: "bob")
+
+    conn =
+      build_conn()
+      |> assign(:user, admin)
+      |> get("/api/pleroma/admin/users/search?query=bo")
+
+    assert json_response(conn, 200) == %{
+             "count" => 1,
+             "page_size" => 50,
+             "users" => [
+               %{
+                 "deactivated" => user.info.deactivated,
+                 "id" => user.id,
+                 "nickname" => user.nickname
+               }
+             ]
+           }
+  end
 end

From 388a3f4ca2299faca366f6dc19485dc03c307aa5 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Thu, 28 Feb 2019 16:23:24 +0000
Subject: [PATCH 37/68] mastodon websocket: bring back infinity timeout

---
 lib/pleroma/web/mastodon_api/websocket_handler.ex | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/web/mastodon_api/websocket_handler.ex b/lib/pleroma/web/mastodon_api/websocket_handler.ex
index edab961f0..bb7ecad69 100644
--- a/lib/pleroma/web/mastodon_api/websocket_handler.ex
+++ b/lib/pleroma/web/mastodon_api/websocket_handler.ex
@@ -23,13 +23,16 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do
   ]
   @anonymous_streams ["public", "public:local", "hashtag"]
 
-  def init(%{qs: qs} = req, _state) do
+  # Handled by periodic keepalive in Pleroma.Web.Streamer.
+  @timeout :infinity
+
+  def init(%{qs: qs} = req, state) do
     with params <- :cow_qs.parse_qs(qs),
          access_token <- List.keyfind(params, "access_token", 0),
          {_, stream} <- List.keyfind(params, "stream", 0),
          {:ok, user} <- allow_request(stream, access_token),
          topic when is_binary(topic) <- expand_topic(stream, params) do
-      {:cowboy_websocket, req, %{user: user, topic: topic}}
+      {:cowboy_websocket, req, %{user: user, topic: topic}, %{idle_timeout: @timeout}}
     else
       {:error, code} ->
         Logger.debug("#{__MODULE__} denied connection: #{inspect(code)} - #{inspect(req)}")

From 28b40932335e1b31a240f51740ace415ab558f43 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Thu, 28 Feb 2019 16:23:55 +0000
Subject: [PATCH 38/68] mastodon websocket: return errors using ok, not stop

---
 lib/pleroma/web/mastodon_api/websocket_handler.ex | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/web/mastodon_api/websocket_handler.ex b/lib/pleroma/web/mastodon_api/websocket_handler.ex
index bb7ecad69..8efe2efd5 100644
--- a/lib/pleroma/web/mastodon_api/websocket_handler.ex
+++ b/lib/pleroma/web/mastodon_api/websocket_handler.ex
@@ -37,12 +37,12 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do
       {:error, code} ->
         Logger.debug("#{__MODULE__} denied connection: #{inspect(code)} - #{inspect(req)}")
         {:ok, req} = :cowboy_req.reply(code, req)
-        {:stop, req}
+        {:ok, req, state}
 
       error ->
         Logger.debug("#{__MODULE__} denied connection: #{inspect(error)} - #{inspect(req)}")
         {:ok, req} = :cowboy_req.reply(400, req)
-        {:stop, req}
+        {:ok, req, state}
     end
   end
 

From 6b2a1ad1c84b7ef350b0d0be76d6b2e03c633260 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Thu, 28 Feb 2019 16:58:00 +0000
Subject: [PATCH 39/68] config: fix chat endpoint path

---
 config/config.exs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/config/config.exs b/config/config.exs
index 374b4d86b..a620e7451 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -94,8 +94,9 @@ config :pleroma, Pleroma.Web.Endpoint,
       {:_,
        [
          {"/api/v1/streaming", Pleroma.Web.MastodonAPI.WebsocketHandler, []},
-         {"/socket/websocket", Phoenix.Endpoint.CowboyWebsocket,
-          {nil, {Pleroma.Web.Endpoint, Pleroma.Web.UserSocket, websocket_config}}},
+         {"/websocket", Phoenix.Endpoint.CowboyWebSocket,
+          {Phoenix.Transports.WebSocket,
+           {Pleroma.Web.Endpoint, Pleroma.Web.UserSocket, websocket_config}}},
          {:_, Phoenix.Endpoint.Cowboy2Handler, {Pleroma.Web.Endpoint, []}}
        ]}
     ]

From 1341ee650ecde656f385454264f43b21051e86f2 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Fri, 1 Mar 2019 09:37:29 +0300
Subject: [PATCH 40/68] [#675] Do not show DMs in mentions timeline

---
 lib/pleroma/web/activity_pub/activity_pub.ex  | 24 +++++++++++++++++++
 .../web/twitter_api/twitter_api_controller.ex |  1 +
 test/web/activity_pub/activity_pub_test.exs   |  8 +++++++
 .../twitter_api_controller_test.exs           | 19 ++++++++++++++-
 4 files changed, 51 insertions(+), 1 deletion(-)

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index cc255cc9e..a61bfa4db 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -420,6 +420,30 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   @valid_visibilities ~w[direct unlisted public private]
 
+  defp restrict_visibility(query, %{visibility: visibility})
+       when is_list(visibility) do
+    if Enum.all?(visibility, &(&1 in @valid_visibilities)) do
+      query =
+        from(
+          a in query,
+          where:
+            fragment(
+              "activity_visibility(?, ?, ?) = ANY (?)",
+              a.actor,
+              a.recipients,
+              a.data,
+              ^visibility
+            )
+        )
+
+      Ecto.Adapters.SQL.to_sql(:all, Repo, query)
+
+      query
+    else
+      Logger.error("Could not restrict visibility to #{visibility}")
+    end
+  end
+
   defp restrict_visibility(query, %{visibility: visibility})
        when visibility in @valid_visibilities do
     query =
diff --git a/lib/pleroma/web/twitter_api/twitter_api_controller.ex b/lib/pleroma/web/twitter_api/twitter_api_controller.ex
index 0d74c30c3..41e3acc60 100644
--- a/lib/pleroma/web/twitter_api/twitter_api_controller.ex
+++ b/lib/pleroma/web/twitter_api/twitter_api_controller.ex
@@ -167,6 +167,7 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
       params
       |> Map.put("type", ["Create", "Announce", "Follow", "Like"])
       |> Map.put("blocking_user", user)
+      |> Map.put(:visibility, ~w[unlisted public private])
 
     activities = ActivityPub.fetch_activities([user.ap_id], params)
 
diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs
index 11262c523..17f48797b 100644
--- a/test/web/activity_pub/activity_pub_test.exs
+++ b/test/web/activity_pub/activity_pub_test.exs
@@ -55,6 +55,14 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
         ActivityPub.fetch_activities([], %{:visibility => "public", "actor_id" => user.ap_id})
 
       assert activities == [public_activity]
+
+      activities =
+        ActivityPub.fetch_activities([], %{
+          :visibility => ~w[private public],
+          "actor_id" => user.ap_id
+        })
+
+      assert activities == [public_activity, private_activity]
     end
   end
 
diff --git a/test/web/twitter_api/twitter_api_controller_test.exs b/test/web/twitter_api/twitter_api_controller_test.exs
index 05a832967..ed5683779 100644
--- a/test/web/twitter_api/twitter_api_controller_test.exs
+++ b/test/web/twitter_api/twitter_api_controller_test.exs
@@ -427,7 +427,10 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do
 
     test "with credentials", %{conn: conn, user: current_user} do
       {:ok, activity} =
-        ActivityBuilder.insert(%{"to" => [current_user.ap_id]}, %{user: current_user})
+        ActivityBuilder.insert(
+          %{"to" => [current_user.ap_id, "https://www.w3.org/ns/activitystreams#Public"]},
+          %{user: current_user}
+        )
 
       conn =
         conn
@@ -445,6 +448,20 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do
                  mentioned: [current_user]
                })
     end
+
+    test "does not show DMs in mentions timeline", %{conn: conn, user: current_user} do
+      {:ok, _activity} =
+        ActivityBuilder.insert(%{"to" => [current_user.ap_id]}, %{user: current_user})
+
+      conn =
+        conn
+        |> with_credentials(current_user.nickname, "test")
+        |> get("/api/statuses/mentions.json")
+
+      response = json_response(conn, 200)
+
+      assert length(response) == 0
+    end
   end
 
   describe "GET /api/qvitter/statuses/notifications.json" do

From 85734c0d49712d53791237e93637b86b848f0f66 Mon Sep 17 00:00:00 2001
From: eugenijm <eugenijm@protonmail.com>
Date: Fri, 1 Mar 2019 13:25:32 +0300
Subject: [PATCH 41/68] Added migration for setting default tags in existing
 users records

---
 .../20190301101154_add_default_tags_to_user.exs          | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 priv/repo/migrations/20190301101154_add_default_tags_to_user.exs

diff --git a/priv/repo/migrations/20190301101154_add_default_tags_to_user.exs b/priv/repo/migrations/20190301101154_add_default_tags_to_user.exs
new file mode 100644
index 000000000..faeb8f1c6
--- /dev/null
+++ b/priv/repo/migrations/20190301101154_add_default_tags_to_user.exs
@@ -0,0 +1,9 @@
+defmodule Pleroma.Repo.Migrations.AddDefaultTagsToUser do
+  use Ecto.Migration
+
+  def up do
+    execute "UPDATE users SET tags = array[]::varchar[] where tags IS NULL"
+  end
+
+  def down, do: :noop
+end

From 689b0730f8074b283da255d4520d05cc1072557e Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Fri, 1 Mar 2019 12:21:09 +0000
Subject: [PATCH 42/68] activitypub: fix date header format

HTTP date header specification says that days must always be two-digit.
Accordingly, change the format string used to ensure days are always
represented as two-digit (e.g. 01).
---
 lib/pleroma/web/activity_pub/activity_pub.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 52404c7e5..16ae65867 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -828,7 +828,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
     date =
       NaiveDateTime.utc_now()
-      |> Timex.format!("{WDshort}, {D} {Mshort} {YYYY} {h24}:{m}:{s} GMT")
+      |> Timex.format!("{WDshort}, {0D} {Mshort} {YYYY} {h24}:{m}:{s} GMT")
 
     signature =
       Pleroma.Web.HTTPSignatures.sign(actor, %{

From 9c6abec4d8b2487edeb124aa197a5dd6d771e345 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Fri, 1 Mar 2019 15:48:04 +0300
Subject: [PATCH 43/68] use commonapi.post instead of activitybulder

---
 .../web/twitter_api/twitter_api_controller_test.exs | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/test/web/twitter_api/twitter_api_controller_test.exs b/test/web/twitter_api/twitter_api_controller_test.exs
index ed5683779..7125d85ab 100644
--- a/test/web/twitter_api/twitter_api_controller_test.exs
+++ b/test/web/twitter_api/twitter_api_controller_test.exs
@@ -427,10 +427,10 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do
 
     test "with credentials", %{conn: conn, user: current_user} do
       {:ok, activity} =
-        ActivityBuilder.insert(
-          %{"to" => [current_user.ap_id, "https://www.w3.org/ns/activitystreams#Public"]},
-          %{user: current_user}
-        )
+        CommonAPI.post(current_user, %{
+          "status" => "why is tenshi eating a corndog so cute?",
+          "visibility" => "public"
+        })
 
       conn =
         conn
@@ -451,7 +451,10 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do
 
     test "does not show DMs in mentions timeline", %{conn: conn, user: current_user} do
       {:ok, _activity} =
-        ActivityBuilder.insert(%{"to" => [current_user.ap_id]}, %{user: current_user})
+        CommonAPI.post(current_user, %{
+          "status" => "Have you guys ever seen how cute tenshi eating a corndog is?",
+          "visibility" => "direct"
+        })
 
       conn =
         conn

From adac7455122d37058bff2e4a805066f2cd24fbf9 Mon Sep 17 00:00:00 2001
From: Maxim Filippov <colixer@gmail.com>
Date: Fri, 1 Mar 2019 17:34:14 +0300
Subject: [PATCH 44/68] Add docs to /users/search

---
 docs/Admin-API.md | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/docs/Admin-API.md b/docs/Admin-API.md
index 508981d38..ed19bc875 100644
--- a/docs/Admin-API.md
+++ b/docs/Admin-API.md
@@ -20,6 +20,26 @@ Authentication is required and the user must be an admin.
 ]
 ```
 
+## `/api/pleroma/admin/users/search?query={query}`
+
+### Search users
+
+- Method `GET`
+- Params:
+  - `query`
+- Response:
+
+```JSON
+[
+    {
+        "deactivated": bool,
+        "id": integer,
+        "nickname": string
+    },
+    ...
+]
+```
+
 ## `/api/pleroma/admin/user`
 
 ### Remove a user

From 5b08b470f69738f4528455a58fefe3a8d4acae02 Mon Sep 17 00:00:00 2001
From: Maxim Filippov <colixer@gmail.com>
Date: Fri, 1 Mar 2019 20:13:02 +0300
Subject: [PATCH 45/68] Add "local" params to users search

---
 docs/Admin-API.md                             |  7 ++-
 lib/pleroma/user.ex                           | 39 ++++++++----
 .../web/admin_api/admin_api_controller.ex     | 10 ++-
 .../mastodon_api/mastodon_api_controller.ex   |  6 +-
 .../web/twitter_api/twitter_api_controller.ex |  2 +-
 .../admin_api/admin_api_controller_test.exs   | 62 +++++++++++++------
 6 files changed, 87 insertions(+), 39 deletions(-)

diff --git a/docs/Admin-API.md b/docs/Admin-API.md
index ed19bc875..4403620bf 100644
--- a/docs/Admin-API.md
+++ b/docs/Admin-API.md
@@ -20,13 +20,14 @@ Authentication is required and the user must be an admin.
 ]
 ```
 
-## `/api/pleroma/admin/users/search?query={query}`
+## `/api/pleroma/admin/users/search?query={query}&local={local}`
 
-### Search users
+### Search users by name or nickname
 
 - Method `GET`
 - Params:
-  - `query`
+  - `query`: **string** search term
+  - `local`: **bool** whether to return only local users
 - Response:
 
 ```JSON
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 52df171c5..af3ce705d 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -755,18 +755,25 @@ defmodule Pleroma.User do
     Repo.all(query)
   end
 
-  def search(query, resolve \\ false, for_user \\ nil, limit \\ 20) do
+  def search(term, options \\ %{}) do
     # Strip the beginning @ off if there is a query
-    query = String.trim_leading(query, "@")
+    term = String.trim_leading(term, "@")
+    query = options[:query] || User
 
-    if resolve, do: get_or_fetch(query)
+    if options[:resolve], do: get_or_fetch(term)
 
-    fts_results = do_search(fts_search_subquery(query), for_user, %{limit: limit})
+    fts_results =
+      do_search(fts_search_subquery(term, query), options[:for_user], %{
+        limit: options[:limit]
+      })
 
     {:ok, trigram_results} =
       Repo.transaction(fn ->
         Ecto.Adapters.SQL.query(Repo, "select set_limit(0.25)", [])
-        do_search(trigram_search_subquery(query), for_user, %{limit: limit})
+
+        do_search(trigram_search_subquery(term, query), options[:for_user], %{
+          limit: options[:limit]
+        })
       end)
 
     Enum.uniq_by(fts_results ++ trigram_results, & &1.id)
@@ -809,9 +816,9 @@ defmodule Pleroma.User do
     boost_search_results(results, for_user)
   end
 
-  defp fts_search_subquery(query) do
+  defp fts_search_subquery(term, query) do
     processed_query =
-      query
+      term
       |> String.replace(~r/\W+/, " ")
       |> String.trim()
       |> String.split()
@@ -819,7 +826,7 @@ defmodule Pleroma.User do
       |> Enum.join(" | ")
 
     from(
-      u in User,
+      u in query,
       select_merge: %{
         search_rank:
           fragment(
@@ -849,19 +856,19 @@ defmodule Pleroma.User do
     )
   end
 
-  defp trigram_search_subquery(query) do
+  defp trigram_search_subquery(term, query) do
     from(
-      u in User,
+      u in query,
       select_merge: %{
         search_rank:
           fragment(
             "similarity(?, trim(? || ' ' || coalesce(?, '')))",
-            ^query,
+            ^term,
             u.nickname,
             u.name
           )
       },
-      where: fragment("trim(? || ' ' || coalesce(?, '')) % ?", u.nickname, u.name, ^query)
+      where: fragment("trim(? || ' ' || coalesce(?, '')) % ?", u.nickname, u.name, ^term)
     )
   end
 
@@ -1018,6 +1025,14 @@ defmodule Pleroma.User do
     update_and_set_cache(cng)
   end
 
+  def maybe_local_user_query(local) when local == true do
+    local_user_query()
+  end
+
+  def maybe_local_user_query(local) when local == false do
+    User
+  end
+
   def local_user_query do
     from(
       u in User,
diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex
index 37159cd40..a8f9e5012 100644
--- a/lib/pleroma/web/admin_api/admin_api_controller.ex
+++ b/lib/pleroma/web/admin_api/admin_api_controller.ex
@@ -78,8 +78,14 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
            )
   end
 
-  def search_users(%{assigns: %{user: admin}} = conn, %{"query" => query}) do
-    users = User.search(query, true, admin, @users_page_size)
+  def search_users(%{assigns: %{user: admin}} = conn, %{"query" => term} = params) do
+    users =
+      User.search(term,
+        query: User.maybe_local_user_query(params["local"] == "true"),
+        resolve: true,
+        for_user: admin,
+        limit: @users_page_size
+      )
 
     conn
     |> json(
diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
index 12987442a..056be49b0 100644
--- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
+++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
@@ -894,7 +894,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   end
 
   def search2(%{assigns: %{user: user}} = conn, %{"q" => query} = params) do
-    accounts = User.search(query, params["resolve"] == "true", user)
+    accounts = User.search(query, resolve: params["resolve"] == "true", for_user: user)
 
     statuses = status_search(user, query)
 
@@ -919,7 +919,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   end
 
   def search(%{assigns: %{user: user}} = conn, %{"q" => query} = params) do
-    accounts = User.search(query, params["resolve"] == "true", user)
+    accounts = User.search(query, resolve: params["resolve"] == "true", for_user: user)
 
     statuses = status_search(user, query)
 
@@ -941,7 +941,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   end
 
   def account_search(%{assigns: %{user: user}} = conn, %{"q" => query} = params) do
-    accounts = User.search(query, params["resolve"] == "true", user)
+    accounts = User.search(query, resolve: params["resolve"] == "true", for_user: user)
 
     res = AccountView.render("accounts.json", users: accounts, for: user, as: :user)
 
diff --git a/lib/pleroma/web/twitter_api/twitter_api_controller.ex b/lib/pleroma/web/twitter_api/twitter_api_controller.ex
index 0d74c30c3..19264a93f 100644
--- a/lib/pleroma/web/twitter_api/twitter_api_controller.ex
+++ b/lib/pleroma/web/twitter_api/twitter_api_controller.ex
@@ -702,7 +702,7 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
   end
 
   def search_user(%{assigns: %{user: user}} = conn, %{"query" => query}) do
-    users = User.search(query, true, user)
+    users = User.search(query, resolve: true, for_user: user)
 
     conn
     |> put_view(UserView)
diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs
index 14625af32..460f2a6bd 100644
--- a/test/web/admin_api/admin_api_controller_test.exs
+++ b/test/web/admin_api/admin_api_controller_test.exs
@@ -388,25 +388,51 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
              }
   end
 
-  test "GET /api/pleroma/admin/users/search" do
-    admin = insert(:user, info: %{is_admin: true})
-    user = insert(:user, nickname: "bob")
+  describe "GET /api/pleroma/admin/users/search" do
+    test "regular search" do
+      admin = insert(:user, info: %{is_admin: true})
+      user = insert(:user, nickname: "bob")
 
-    conn =
-      build_conn()
-      |> assign(:user, admin)
-      |> get("/api/pleroma/admin/users/search?query=bo")
+      conn =
+        build_conn()
+        |> assign(:user, admin)
+        |> get("/api/pleroma/admin/users/search?query=bo")
 
-    assert json_response(conn, 200) == %{
-             "count" => 1,
-             "page_size" => 50,
-             "users" => [
-               %{
-                 "deactivated" => user.info.deactivated,
-                 "id" => user.id,
-                 "nickname" => user.nickname
-               }
-             ]
-           }
+      assert json_response(conn, 200) == %{
+              "count" => 1,
+              "page_size" => 50,
+              "users" => [
+                %{
+                  "deactivated" => user.info.deactivated,
+                  "id" => user.id,
+                  "nickname" => user.nickname
+                }
+              ]
+            }
+    end
+
+    test "only local users" do
+      admin = insert(:user, info: %{is_admin: true})
+      user = insert(:user, nickname: "bob")
+
+      insert(:user, nickname: "bobb", local: false)
+
+      conn =
+        build_conn()
+        |> assign(:user, admin)
+        |> get("/api/pleroma/admin/users/search?query=bo&local=true")
+
+      assert json_response(conn, 200) == %{
+              "count" => 1,
+              "page_size" => 50,
+              "users" => [
+                %{
+                  "deactivated" => user.info.deactivated,
+                  "id" => user.id,
+                  "nickname" => user.nickname
+                }
+              ]
+            }
+    end
   end
 end

From f1a4c3163b18692a7a8bd9874a45e75a6535dd5a Mon Sep 17 00:00:00 2001
From: Maxim Filippov <colixer@gmail.com>
Date: Fri, 1 Mar 2019 20:23:03 +0300
Subject: [PATCH 46/68] Show current user in users list as well

---
 lib/pleroma/user.ex                           |  3 +-
 .../web/admin_api/admin_api_controller.ex     |  2 +-
 .../admin_api/admin_api_controller_test.exs   | 45 ++++++++++---------
 3 files changed, 27 insertions(+), 23 deletions(-)

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index af3ce705d..37f8da892 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -779,10 +779,9 @@ defmodule Pleroma.User do
     Enum.uniq_by(fts_results ++ trigram_results, & &1.id)
   end
 
-  def all_except_one(user, page, page_size) do
+  def all(page, page_size) do
     from(
       u in User,
-      where: u.id != ^user.id,
       limit: ^page_size,
       offset: ^((page - 1) * page_size),
       order_by: u.id
diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex
index a8f9e5012..270097d35 100644
--- a/lib/pleroma/web/admin_api/admin_api_controller.ex
+++ b/lib/pleroma/web/admin_api/admin_api_controller.ex
@@ -65,7 +65,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
 
   def list_users(%{assigns: %{user: admin}} = conn, %{"page" => page_string}) do
     with {page, _} <- Integer.parse(page_string),
-         users <- User.all_except_one(admin, page, @users_page_size),
+         users <- User.all(page, @users_page_size),
          count <- User.count_all_except_one(admin),
          do:
            conn
diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs
index 460f2a6bd..0679f5dfe 100644
--- a/test/web/admin_api/admin_api_controller_test.exs
+++ b/test/web/admin_api/admin_api_controller_test.exs
@@ -345,6 +345,11 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
                "count" => 1,
                "page_size" => 50,
                "users" => [
+                %{
+                  "deactivated" => admin.info.deactivated,
+                  "id" => admin.id,
+                  "nickname" => admin.nickname
+                },
                  %{
                    "deactivated" => user.info.deactivated,
                    "id" => user.id,
@@ -399,16 +404,16 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
         |> get("/api/pleroma/admin/users/search?query=bo")
 
       assert json_response(conn, 200) == %{
-              "count" => 1,
-              "page_size" => 50,
-              "users" => [
-                %{
-                  "deactivated" => user.info.deactivated,
-                  "id" => user.id,
-                  "nickname" => user.nickname
-                }
-              ]
-            }
+               "count" => 1,
+               "page_size" => 50,
+               "users" => [
+                 %{
+                   "deactivated" => user.info.deactivated,
+                   "id" => user.id,
+                   "nickname" => user.nickname
+                 }
+               ]
+             }
     end
 
     test "only local users" do
@@ -423,16 +428,16 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
         |> get("/api/pleroma/admin/users/search?query=bo&local=true")
 
       assert json_response(conn, 200) == %{
-              "count" => 1,
-              "page_size" => 50,
-              "users" => [
-                %{
-                  "deactivated" => user.info.deactivated,
-                  "id" => user.id,
-                  "nickname" => user.nickname
-                }
-              ]
-            }
+               "count" => 1,
+               "page_size" => 50,
+               "users" => [
+                 %{
+                   "deactivated" => user.info.deactivated,
+                   "id" => user.id,
+                   "nickname" => user.nickname
+                 }
+               ]
+             }
     end
   end
 end

From f384a9a2563d2fb6161ebb824534fda08cf67c64 Mon Sep 17 00:00:00 2001
From: Maxim Filippov <colixer@gmail.com>
Date: Fri, 1 Mar 2019 20:23:19 +0300
Subject: [PATCH 47/68] Format

---
 test/web/admin_api/admin_api_controller_test.exs | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs
index 0679f5dfe..a3042fa05 100644
--- a/test/web/admin_api/admin_api_controller_test.exs
+++ b/test/web/admin_api/admin_api_controller_test.exs
@@ -345,11 +345,11 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
                "count" => 1,
                "page_size" => 50,
                "users" => [
-                %{
-                  "deactivated" => admin.info.deactivated,
-                  "id" => admin.id,
-                  "nickname" => admin.nickname
-                },
+                 %{
+                   "deactivated" => admin.info.deactivated,
+                   "id" => admin.id,
+                   "nickname" => admin.nickname
+                 },
                  %{
                    "deactivated" => user.info.deactivated,
                    "id" => user.id,

From aaa9fed1ca196b55d8c05af838d6947959548bf0 Mon Sep 17 00:00:00 2001
From: Maxim Filippov <colixer@gmail.com>
Date: Fri, 1 Mar 2019 20:58:47 +0300
Subject: [PATCH 48/68] Fix user_test

---
 test/user_test.exs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/user_test.exs b/test/user_test.exs
index 0b1c39ecf..27137fc35 100644
--- a/test/user_test.exs
+++ b/test/user_test.exs
@@ -901,7 +901,7 @@ defmodule Pleroma.UserTest do
       {:ok, follower} = User.follow(follower, u1)
       {:ok, u1} = User.follow(u1, friend)
 
-      assert [friend.id, follower.id, u2.id] == Enum.map(User.search("doe", false, u1), & &1.id)
+      assert [friend.id, follower.id, u2.id] == Enum.map(User.search("doe", resolve: false, for_user: u1), & &1.id)
     end
 
     test "finds a user whose name is nil" do
@@ -923,7 +923,7 @@ defmodule Pleroma.UserTest do
     end
 
     test "works with URIs" do
-      results = User.search("http://mastodon.example.org/users/admin", true)
+      results = User.search("http://mastodon.example.org/users/admin", resolve: true)
       result = results |> List.first()
 
       user = User.get_by_ap_id("http://mastodon.example.org/users/admin")

From a25c1313aee2f5f3d2b858b9802698f29acf4043 Mon Sep 17 00:00:00 2001
From: Maxim Filippov <colixer@gmail.com>
Date: Fri, 1 Mar 2019 21:07:05 +0300
Subject: [PATCH 49/68] Format

---
 test/user_test.exs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/test/user_test.exs b/test/user_test.exs
index 27137fc35..4f9de4e31 100644
--- a/test/user_test.exs
+++ b/test/user_test.exs
@@ -901,7 +901,8 @@ defmodule Pleroma.UserTest do
       {:ok, follower} = User.follow(follower, u1)
       {:ok, u1} = User.follow(u1, friend)
 
-      assert [friend.id, follower.id, u2.id] == Enum.map(User.search("doe", resolve: false, for_user: u1), & &1.id)
+      assert [friend.id, follower.id, u2.id] ==
+               Enum.map(User.search("doe", resolve: false, for_user: u1), & &1.id)
     end
 
     test "finds a user whose name is nil" do

From f635b675b2cc0bc10b395cd71ae1720b0696d364 Mon Sep 17 00:00:00 2001
From: Maxim Filippov <colixer@gmail.com>
Date: Fri, 1 Mar 2019 21:17:20 +0300
Subject: [PATCH 50/68] Refactor a little bit

---
 lib/pleroma/user.ex                              | 16 ++++------------
 .../web/admin_api/admin_api_controller.ex        |  8 ++++----
 2 files changed, 8 insertions(+), 16 deletions(-)

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 37f8da892..3c6fb4f9b 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -763,17 +763,13 @@ defmodule Pleroma.User do
     if options[:resolve], do: get_or_fetch(term)
 
     fts_results =
-      do_search(fts_search_subquery(term, query), options[:for_user], %{
-        limit: options[:limit]
-      })
+      do_search(fts_search_subquery(term, query), options[:for_user], limit: options[:limit])
 
     {:ok, trigram_results} =
       Repo.transaction(fn ->
         Ecto.Adapters.SQL.query(Repo, "select set_limit(0.25)", [])
 
-        do_search(trigram_search_subquery(term, query), options[:for_user], %{
-          limit: options[:limit]
-        })
+        do_search(trigram_search_subquery(term, query), options[:for_user], limit: options[:limit])
       end)
 
     Enum.uniq_by(fts_results ++ trigram_results, & &1.id)
@@ -1024,12 +1020,8 @@ defmodule Pleroma.User do
     update_and_set_cache(cng)
   end
 
-  def maybe_local_user_query(local) when local == true do
-    local_user_query()
-  end
-
-  def maybe_local_user_query(local) when local == false do
-    User
+  def maybe_local_user_query(local) do
+    if local, do: local_user_query(), else: User
   end
 
   def local_user_query do
diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex
index 270097d35..33f9689cd 100644
--- a/lib/pleroma/web/admin_api/admin_api_controller.ex
+++ b/lib/pleroma/web/admin_api/admin_api_controller.ex
@@ -70,11 +70,11 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
          do:
            conn
            |> json(
-             AccountView.render("index.json", %{
+             AccountView.render("index.json",
                users: users,
                count: count,
                page_size: @users_page_size
-             })
+             )
            )
   end
 
@@ -89,11 +89,11 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
 
     conn
     |> json(
-      AccountView.render("index.json", %{
+      AccountView.render("index.json",
         users: users,
         count: length(users),
         page_size: @users_page_size
-      })
+      )
     )
   end
 

From b6a001a34c09a35070049065367ccb7f0382a1e1 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Sat, 2 Mar 2019 04:04:16 +0100
Subject: [PATCH 51/68] Web.OAuth.OAuthController: Fix scopes Enum.join for
 OAuth response

---
 lib/pleroma/web/oauth/oauth_controller.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/oauth/oauth_controller.ex b/lib/pleroma/web/oauth/oauth_controller.ex
index 7c1a3adbd..4309cf58f 100644
--- a/lib/pleroma/web/oauth/oauth_controller.ex
+++ b/lib/pleroma/web/oauth/oauth_controller.ex
@@ -114,7 +114,7 @@ defmodule Pleroma.Web.OAuth.OAuthController do
         refresh_token: token.refresh_token,
         created_at: DateTime.to_unix(inserted_at),
         expires_in: 60 * 10,
-        scope: Enum.join(token.scopes)
+        scope: Enum.join(token.scopes, " ")
       }
 
       json(conn, response)

From bb9e40968a90b882ba85e71a2622756e40563207 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Sat, 2 Mar 2019 04:10:43 +0100
Subject: [PATCH 52/68] Web.OAuth.OAuthControllerTest: Add test against token
 formatting

---
 test/web/oauth/oauth_controller_test.exs | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/test/web/oauth/oauth_controller_test.exs b/test/web/oauth/oauth_controller_test.exs
index 53d83e6e8..ed94416ff 100644
--- a/test/web/oauth/oauth_controller_test.exs
+++ b/test/web/oauth/oauth_controller_test.exs
@@ -165,10 +165,10 @@ defmodule Pleroma.Web.OAuth.OAuthControllerTest do
 
   test "issues a token for request with HTTP basic auth client credentials" do
     user = insert(:user)
-    app = insert(:oauth_app, scopes: ["scope1", "scope2"])
+    app = insert(:oauth_app, scopes: ["scope1", "scope2", "scope3"])
 
-    {:ok, auth} = Authorization.create_authorization(app, user, ["scope2"])
-    assert auth.scopes == ["scope2"]
+    {:ok, auth} = Authorization.create_authorization(app, user, ["scope1", "scope2"])
+    assert auth.scopes == ["scope1", "scope2"]
 
     app_encoded =
       (URI.encode_www_form(app.client_id) <> ":" <> URI.encode_www_form(app.client_secret))
@@ -183,11 +183,13 @@ defmodule Pleroma.Web.OAuth.OAuthControllerTest do
         "redirect_uri" => app.redirect_uris
       })
 
-    assert %{"access_token" => token} = json_response(conn, 200)
+    assert %{"access_token" => token, "scope" => scope} = json_response(conn, 200)
+
+    assert scope == "scope1 scope2"
 
     token = Repo.get_by(Token, token: token)
     assert token
-    assert token.scopes == ["scope2"]
+    assert token.scopes == ["scope1", "scope2"]
   end
 
   test "rejects token exchange with invalid client credentials" do

From 56d4e3901217251b5fb19d3e688f376f5a8627c1 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Mon, 11 Feb 2019 22:27:02 +0100
Subject: [PATCH 53/68] Pleroma.User: Add rel=me to URLs where it linkbacks to
 the profile

---
 lib/pleroma/user.ex       |  9 +++++++-
 lib/pleroma/web/rel_me.ex | 46 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 54 insertions(+), 1 deletion(-)
 create mode 100644 lib/pleroma/web/rel_me.ex

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index d58274508..b54613274 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -22,6 +22,7 @@ defmodule Pleroma.User do
   alias Pleroma.Web.OAuth
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.ActivityPub.ActivityPub
+  alias Pleroma.Web.RelMe
 
   require Logger
 
@@ -1202,8 +1203,14 @@ defmodule Pleroma.User do
         {String.trim(name, ":"), url}
       end)
 
+    # TODO: get profile URLs other than user.ap_id
+    profile_urls = user[:ap_id]
+
     bio
-    |> CommonUtils.format_input("text/plain", mentions_format: :full)
+    |> CommonUtils.format_input("text/plain", %{
+      mentions_format: :full,
+      rel: &RelMe.maybe_put_rel_me(&1, profile_urls)
+    })
     |> elem(0)
     |> Formatter.emojify(emoji)
   end
diff --git a/lib/pleroma/web/rel_me.ex b/lib/pleroma/web/rel_me.ex
new file mode 100644
index 000000000..b23c49977
--- /dev/null
+++ b/lib/pleroma/web/rel_me.ex
@@ -0,0 +1,46 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.RelMe do
+  @hackney_options [
+    pool: :media,
+    timeout: 2_000,
+    recv_timeout: 2_000,
+    max_body: 2_000_000
+  ]
+
+  def parse(nil), do: {:error, "No URL provided"}
+
+  if Mix.env() == :test do
+    def parse(url), do: parse_url(url)
+  else
+    def parse(url) do
+      Cachex.fetch!(:rel_me_cache, url, fn _ ->
+        {:commit, parse_url(url)}
+      end)
+    rescue
+      e -> {:error, "Cachex error: #{inspect(e)}"}
+    end
+  end
+
+  defp parse_url(url) do
+    {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: @hackney_options)
+
+    Floki.attribute(html, "link[rel=me]", "href") ++ Floki.attribute(html, "a[rel=me]", "href")
+  rescue
+    e -> {:error, "Parsing error: #{inspect(e)}"}
+  end
+
+  def maybe_put_rel_me("http" <> _ = target_page, urls) when not is_nil(urls) do
+    if Enum.any?(parse(target_page), fn x -> x in urls end) do
+      "rel=\"me\" "
+    else
+      ""
+    end
+  end
+
+  def maybe_put_rel_me(_, _) do
+    ""
+  end
+end

From 25e588496adab62be79da65e8dc23426b8813159 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Sat, 2 Mar 2019 06:13:04 +0100
Subject: [PATCH 54/68] Pleroma.Web.RelMeTest: Add test against
 Pleroma.Web.RelMe

---
 test/fixtures/rel_me_anchor.html | 14 +++++++++
 test/fixtures/rel_me_link.html   | 14 +++++++++
 test/fixtures/rel_me_null.html   | 13 ++++++++
 test/web/rel_me_test.exs         | 54 ++++++++++++++++++++++++++++++++
 4 files changed, 95 insertions(+)
 create mode 100644 test/fixtures/rel_me_anchor.html
 create mode 100644 test/fixtures/rel_me_link.html
 create mode 100644 test/fixtures/rel_me_null.html
 create mode 100644 test/web/rel_me_test.exs

diff --git a/test/fixtures/rel_me_anchor.html b/test/fixtures/rel_me_anchor.html
new file mode 100644
index 000000000..5abcce129
--- /dev/null
+++ b/test/fixtures/rel_me_anchor.html
@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html>
+	<head>
+		<meta charset="utf-8"/>
+		<title>Blog</title>
+	</head>
+	<body>
+	<article>
+		<h1>Lorem ipsum</h1>
+		<p>Lorem ipsum dolor sit ameph, …</p>
+		<a rel="me" href="https://social.example.org/users/lain">lain’s account</a>
+	</article>
+	</body>
+</html>
diff --git a/test/fixtures/rel_me_link.html b/test/fixtures/rel_me_link.html
new file mode 100644
index 000000000..b9ff18f6e
--- /dev/null
+++ b/test/fixtures/rel_me_link.html
@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html>
+	<head>
+		<meta charset="utf-8"/>
+		<title>Blog</title>
+		<link rel="me" href="https://social.example.org/users/lain"/>
+	</head>
+	<body>
+	<article>
+		<h1>Lorem ipsum</h1>
+		<p>Lorem ipsum dolor sit ameph, …</p>
+	</article>
+	</body>
+</html>
diff --git a/test/fixtures/rel_me_null.html b/test/fixtures/rel_me_null.html
new file mode 100644
index 000000000..57d424b80
--- /dev/null
+++ b/test/fixtures/rel_me_null.html
@@ -0,0 +1,13 @@
+<!DOCTYPE html>
+<html>
+	<head>
+		<meta charset="utf-8"/>
+		<title>Blog</title>
+	</head>
+	<body>
+	<article>
+		<h1>Lorem ipsum</h1>
+		<p>Lorem ipsum dolor sit ameph, …</p>
+	</article>
+	</body>
+</html>
diff --git a/test/web/rel_me_test.exs b/test/web/rel_me_test.exs
new file mode 100644
index 000000000..94cc01728
--- /dev/null
+++ b/test/web/rel_me_test.exs
@@ -0,0 +1,54 @@
+defmodule Pleroma.Web.RelMeTest do
+  use ExUnit.Case, async: true
+
+  setup do
+    Tesla.Mock.mock(fn
+      %{
+        method: :get,
+        url: "http://example.com/rel_me/anchor"
+      } ->
+        %Tesla.Env{status: 200, body: File.read!("test/fixtures/rel_me_anchor.html")}
+
+      %{
+        method: :get,
+        url: "http://example.com/rel_me/link"
+      } ->
+        %Tesla.Env{status: 200, body: File.read!("test/fixtures/rel_me_link.html")}
+
+      %{
+        method: :get,
+        url: "http://example.com/rel_me/null"
+      } ->
+        %Tesla.Env{status: 200, body: File.read!("test/fixtures/rel_me_null.html")}
+    end)
+
+    :ok
+  end
+
+  test "parse/1" do
+    hrefs = ["https://social.example.org/users/lain"]
+
+    assert Pleroma.Web.RelMe.parse("http://example.com/rel_me/null") == {:ok, []}
+    assert {:error, _} = Pleroma.Web.RelMe.parse("http://example.com/rel_me/error")
+
+    assert Pleroma.Web.RelMe.parse("http://example.com/rel_me/link") == {:ok, hrefs}
+    assert Pleroma.Web.RelMe.parse("http://example.com/rel_me/anchor") == {:ok, hrefs}
+  end
+
+  test "maybe_put_rel_me/2" do
+    profile_urls = ["https://social.example.org/users/lain"]
+    attr = "rel=\"me\" "
+
+    assert Pleroma.Web.RelMe.maybe_put_rel_me("http://example.com/rel_me/null", profile_urls) ==
+             ""
+
+    assert Pleroma.Web.RelMe.maybe_put_rel_me("http://example.com/rel_me/error", profile_urls) ==
+             ""
+
+    assert Pleroma.Web.RelMe.maybe_put_rel_me("http://example.com/rel_me/anchor", profile_urls) ==
+             attr
+
+    assert Pleroma.Web.RelMe.maybe_put_rel_me("http://example.com/rel_me/link", profile_urls) ==
+             attr
+  end
+end

From 9b83236fb0292ae19a981b8f464183f8c6214a48 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Sat, 2 Mar 2019 06:32:03 +0100
Subject: [PATCH 55/68] Pleroma.Web.RelMe: fixups

---
 lib/pleroma/web/rel_me.ex | 27 ++++++++++++++++-----------
 1 file changed, 16 insertions(+), 11 deletions(-)

diff --git a/lib/pleroma/web/rel_me.ex b/lib/pleroma/web/rel_me.ex
index b23c49977..3f0ee9ac6 100644
--- a/lib/pleroma/web/rel_me.ex
+++ b/lib/pleroma/web/rel_me.ex
@@ -10,12 +10,10 @@ defmodule Pleroma.Web.RelMe do
     max_body: 2_000_000
   ]
 
-  def parse(nil), do: {:error, "No URL provided"}
-
   if Mix.env() == :test do
-    def parse(url), do: parse_url(url)
+    def parse(url) when is_binary(url), do: parse_url(url)
   else
-    def parse(url) do
+    def parse(url) when is_binary(url) do
       Cachex.fetch!(:rel_me_cache, url, fn _ ->
         {:commit, parse_url(url)}
       end)
@@ -24,20 +22,27 @@ defmodule Pleroma.Web.RelMe do
     end
   end
 
+  def parse(_), do: {:error, "No URL provided"}
+
   defp parse_url(url) do
     {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: @hackney_options)
 
-    Floki.attribute(html, "link[rel=me]", "href") ++ Floki.attribute(html, "a[rel=me]", "href")
+    data =
+      Floki.attribute(html, "link[rel=me]", "href") ++ Floki.attribute(html, "a[rel=me]", "href")
+
+    {:ok, data}
   rescue
     e -> {:error, "Parsing error: #{inspect(e)}"}
   end
 
-  def maybe_put_rel_me("http" <> _ = target_page, urls) when not is_nil(urls) do
-    if Enum.any?(parse(target_page), fn x -> x in urls end) do
-      "rel=\"me\" "
-    else
-      ""
-    end
+  def maybe_put_rel_me("http" <> _ = target_page, profile_urls) when is_list(profile_urls) do
+    {:ok, rel_me_hrefs} = parse(target_page)
+
+    true = Enum.any?(rel_me_hrefs, fn x -> x in profile_urls end)
+
+    "rel=\"me\" "
+  rescue
+    _ -> ""
   end
 
   def maybe_put_rel_me(_, _) do

From 3d22642352b797c6963dba7c9116c27699d5c641 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Sat, 2 Mar 2019 06:33:15 +0100
Subject: [PATCH 56/68] Pleroma.User: Pass an array to profile_urls

---
 lib/pleroma/user.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index b54613274..447beb25b 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -1204,7 +1204,7 @@ defmodule Pleroma.User do
       end)
 
     # TODO: get profile URLs other than user.ap_id
-    profile_urls = user[:ap_id]
+    profile_urls = [user[:ap_id]]
 
     bio
     |> CommonUtils.format_input("text/plain", %{

From 39a5bea9b73daa63203b0640d0a7906d6f1af143 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Sat, 2 Mar 2019 06:57:28 +0100
Subject: [PATCH 57/68] Pleroma.User: Fix syntax and user.ap_id call

---
 lib/pleroma/user.ex | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 447beb25b..a9592df6d 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -1204,13 +1204,13 @@ defmodule Pleroma.User do
       end)
 
     # TODO: get profile URLs other than user.ap_id
-    profile_urls = [user[:ap_id]]
+    profile_urls = [user.ap_id]
 
     bio
-    |> CommonUtils.format_input("text/plain", %{
+    |> CommonUtils.format_input("text/plain", [
       mentions_format: :full,
       rel: &RelMe.maybe_put_rel_me(&1, profile_urls)
-    })
+    ])
     |> elem(0)
     |> Formatter.emojify(emoji)
   end

From 8e6f7fdb86bc7f90618c59ee4b13b30b94fda475 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Sat, 2 Mar 2019 06:58:42 +0100
Subject: [PATCH 58/68] RelMe.maybe_put_rel_me/2: When true put "me" otherwise
 nil

---
 lib/pleroma/web/rel_me.ex | 6 +++---
 test/web/rel_me_test.exs  | 7 ++++---
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/lib/pleroma/web/rel_me.ex b/lib/pleroma/web/rel_me.ex
index 3f0ee9ac6..a07db966f 100644
--- a/lib/pleroma/web/rel_me.ex
+++ b/lib/pleroma/web/rel_me.ex
@@ -40,12 +40,12 @@ defmodule Pleroma.Web.RelMe do
 
     true = Enum.any?(rel_me_hrefs, fn x -> x in profile_urls end)
 
-    "rel=\"me\" "
+    "me"
   rescue
-    _ -> ""
+    _ -> nil
   end
 
   def maybe_put_rel_me(_, _) do
-    ""
+    nil
   end
 end
diff --git a/test/web/rel_me_test.exs b/test/web/rel_me_test.exs
index 94cc01728..ba8038e69 100644
--- a/test/web/rel_me_test.exs
+++ b/test/web/rel_me_test.exs
@@ -37,13 +37,14 @@ defmodule Pleroma.Web.RelMeTest do
 
   test "maybe_put_rel_me/2" do
     profile_urls = ["https://social.example.org/users/lain"]
-    attr = "rel=\"me\" "
+    attr = "me"
+    fallback = nil
 
     assert Pleroma.Web.RelMe.maybe_put_rel_me("http://example.com/rel_me/null", profile_urls) ==
-             ""
+             fallback
 
     assert Pleroma.Web.RelMe.maybe_put_rel_me("http://example.com/rel_me/error", profile_urls) ==
-             ""
+             fallback
 
     assert Pleroma.Web.RelMe.maybe_put_rel_me("http://example.com/rel_me/anchor", profile_urls) ==
              attr

From 7b9868f34344144bfb965cdd099f71b2617976c6 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Sat, 2 Mar 2019 06:59:09 +0100
Subject: [PATCH 59/68] Pleroma.UserTest: Add tests for rel=me

---
 test/user_test.exs | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/test/user_test.exs b/test/user_test.exs
index cbe4693fc..e182a809f 100644
--- a/test/user_test.exs
+++ b/test/user_test.exs
@@ -1039,6 +1039,22 @@ defmodule Pleroma.UserTest do
 
       assert expected_text == User.parse_bio(bio, user)
     end
+
+    test "Adds rel=me on linkbacked urls" do
+      user = insert(:user, ap_id: "http://social.example.org/users/lain")
+
+      bio = "http://example.org/rel_me/null"
+      expected_text = "<a href=\"#{bio}\">#{bio}</a>"
+      assert expected_text == User.parse_bio(bio, user)
+
+      bio = "http://example.org/rel_me/link"
+      expected_text = "<a href=\"#{bio}\">#{bio}</a>"
+      assert expected_text == User.parse_bio(bio, user)
+
+      bio = "http://example.org/rel_me/anchor"
+      expected_text = "<a href=\"#{bio}\">#{bio}</a>"
+      assert expected_text == User.parse_bio(bio, user)
+    end
   end
 
   test "bookmarks" do

From f2452d5700afc48284638ae2cd1bff4886571d8c Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Sat, 2 Mar 2019 07:04:49 +0100
Subject: [PATCH 60/68] Pleroma.User: mix format

---
 lib/pleroma/user.ex | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index a9592df6d..6ec1033a5 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -1207,10 +1207,10 @@ defmodule Pleroma.User do
     profile_urls = [user.ap_id]
 
     bio
-    |> CommonUtils.format_input("text/plain", [
+    |> CommonUtils.format_input("text/plain",
       mentions_format: :full,
       rel: &RelMe.maybe_put_rel_me(&1, profile_urls)
-    ])
+    )
     |> elem(0)
     |> Formatter.emojify(emoji)
   end

From 2ec8cf566569912b767e15ab467cadd04fd1fd1c Mon Sep 17 00:00:00 2001
From: Maxim Filippov <colixer@gmail.com>
Date: Sat, 2 Mar 2019 17:21:18 +0300
Subject: [PATCH 61/68] Add pagination to search

---
 lib/pleroma/user.ex                           | 110 ++++++++++--------
 .../web/admin_api/admin_api_controller.ex     |  66 +++++++----
 .../admin_api/admin_api_controller_test.exs   |  46 +++++++-
 3 files changed, 152 insertions(+), 70 deletions(-)

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 3c6fb4f9b..230155c33 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -547,11 +547,8 @@ defmodule Pleroma.User do
   end
 
   def get_followers_query(user, page) do
-    from(
-      u in get_followers_query(user, nil),
-      limit: 20,
-      offset: ^((page - 1) * 20)
-    )
+    from(u in get_followers_query(user, nil))
+    |> paginate(page, 20)
   end
 
   def get_followers_query(user), do: get_followers_query(user, nil)
@@ -577,11 +574,8 @@ defmodule Pleroma.User do
   end
 
   def get_friends_query(user, page) do
-    from(
-      u in get_friends_query(user, nil),
-      limit: 20,
-      offset: ^((page - 1) * 20)
-    )
+    from(u in get_friends_query(user, nil))
+    |> paginate(page, 20)
   end
 
   def get_friends_query(user), do: get_friends_query(user, nil)
@@ -755,47 +749,64 @@ defmodule Pleroma.User do
     Repo.all(query)
   end
 
-  def search(term, options \\ %{}) do
-    # Strip the beginning @ off if there is a query
+  @spec search_for_admin(binary(), %{
+          admin: Pleroma.User.t(),
+          local: boolean(),
+          page: number(),
+          page_size: number()
+        }) :: {:ok, [Pleroma.User.t()], number()}
+  def search_for_admin(term, %{admin: admin, local: local, page: page, page_size: page_size}) do
     term = String.trim_leading(term, "@")
-    query = options[:query] || User
 
-    if options[:resolve], do: get_or_fetch(term)
+    local_paginated_query =
+      User
+      |> maybe_local_user_query(local)
+      |> paginate(page, page_size)
 
-    fts_results =
-      do_search(fts_search_subquery(term, query), options[:for_user], limit: options[:limit])
+    search_query = fts_search_subquery(term, local_paginated_query)
+
+    count =
+      term
+      |> fts_search_subquery()
+      |> maybe_local_user_query(local)
+      |> Repo.aggregate(:count, :id)
+
+    {:ok, do_search(search_query, admin), count}
+  end
+
+  @spec all_for_admin(number(), number()) :: {:ok, [Pleroma.User.t()], number()}
+  def all_for_admin(page, page_size) do
+    query = from(u in User, order_by: u.id)
+
+    paginated_query =
+      query
+      |> paginate(page, page_size)
+
+    count =
+      query
+      |> Repo.aggregate(:count, :id)
+
+    {:ok, Repo.all(paginated_query), count}
+  end
+
+  def search(query, resolve \\ false, for_user \\ nil) do
+    # Strip the beginning @ off if there is a query
+    query = String.trim_leading(query, "@")
+
+    if resolve, do: get_or_fetch(query)
+
+    fts_results = do_search(fts_search_subquery(query), for_user)
 
     {:ok, trigram_results} =
       Repo.transaction(fn ->
         Ecto.Adapters.SQL.query(Repo, "select set_limit(0.25)", [])
-
-        do_search(trigram_search_subquery(term, query), options[:for_user], limit: options[:limit])
+        do_search(trigram_search_subquery(query), for_user)
       end)
 
     Enum.uniq_by(fts_results ++ trigram_results, & &1.id)
   end
 
-  def all(page, page_size) do
-    from(
-      u in User,
-      limit: ^page_size,
-      offset: ^((page - 1) * page_size),
-      order_by: u.id
-    )
-    |> Repo.all()
-  end
-
-  def count_all_except_one(user) do
-    query =
-      from(
-        u in User,
-        where: u.id != ^user.id
-      )
-
-    Repo.aggregate(query, :count, :id)
-  end
-
-  defp do_search(subquery, for_user, options) do
+  defp do_search(subquery, for_user, options \\ []) do
     q =
       from(
         s in subquery(subquery),
@@ -811,7 +822,7 @@ defmodule Pleroma.User do
     boost_search_results(results, for_user)
   end
 
-  defp fts_search_subquery(term, query) do
+  defp fts_search_subquery(term, query \\ User) do
     processed_query =
       term
       |> String.replace(~r/\W+/, " ")
@@ -851,9 +862,9 @@ defmodule Pleroma.User do
     )
   end
 
-  defp trigram_search_subquery(term, query) do
+  defp trigram_search_subquery(term) do
     from(
-      u in query,
+      u in User,
       select_merge: %{
         search_rank:
           fragment(
@@ -1020,13 +1031,13 @@ defmodule Pleroma.User do
     update_and_set_cache(cng)
   end
 
-  def maybe_local_user_query(local) do
-    if local, do: local_user_query(), else: User
+  def maybe_local_user_query(query, local) do
+    if local, do: local_user_query(query), else: query
   end
 
-  def local_user_query do
+  def local_user_query(query \\ User) do
     from(
-      u in User,
+      u in query,
       where: u.local == true,
       where: not is_nil(u.nickname)
     )
@@ -1318,4 +1329,11 @@ defmodule Pleroma.User do
     )
     |> Repo.all()
   end
+
+  defp paginate(query, page, page_size) do
+    from(u in query,
+      limit: ^page_size,
+      offset: ^((page - 1) * page_size)
+    )
+  end
 end
diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex
index 33f9689cd..aae02cab8 100644
--- a/lib/pleroma/web/admin_api/admin_api_controller.ex
+++ b/lib/pleroma/web/admin_api/admin_api_controller.ex
@@ -63,38 +63,40 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
          do: json_response(conn, :no_content, "")
   end
 
-  def list_users(%{assigns: %{user: admin}} = conn, %{"page" => page_string}) do
-    with {page, _} <- Integer.parse(page_string),
-         users <- User.all(page, @users_page_size),
-         count <- User.count_all_except_one(admin),
+  def list_users(conn, params) do
+    {page, page_size} = page_params(params)
+
+    with {:ok, users, count} <- User.all_for_admin(page, page_size),
          do:
            conn
            |> json(
              AccountView.render("index.json",
                users: users,
                count: count,
-               page_size: @users_page_size
+               page_size: page_size
              )
            )
   end
 
-  def search_users(%{assigns: %{user: admin}} = conn, %{"query" => term} = params) do
-    users =
-      User.search(term,
-        query: User.maybe_local_user_query(params["local"] == "true"),
-        resolve: true,
-        for_user: admin,
-        limit: @users_page_size
-      )
+  def search_users(%{assigns: %{user: admin}} = conn, %{"query" => query} = params) do
+    {page, page_size} = page_params(params)
 
-    conn
-    |> json(
-      AccountView.render("index.json",
-        users: users,
-        count: length(users),
-        page_size: @users_page_size
-      )
-    )
+    with {:ok, users, count} <-
+           User.search_for_admin(query, %{
+             admin: admin,
+             local: params["local"] == "true",
+             page: page,
+             page_size: page_size
+           }),
+         do:
+           conn
+           |> json(
+             AccountView.render("index.json",
+               users: users,
+               count: count,
+               page_size: page_size
+             )
+           )
   end
 
   def right_add(conn, %{"permission_group" => permission_group, "nickname" => nickname})
@@ -240,4 +242,26 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     |> put_status(500)
     |> json("Something went wrong")
   end
+
+  defp page_params(params) do
+    {get_page(params["page"]), get_page_size(params["page_size"])}
+  end
+
+  defp get_page(page_string) when is_nil(page_string), do: 1
+
+  defp get_page(page_string) do
+    case Integer.parse(page_string) do
+      {page, _} -> page
+      :error -> 1
+    end
+  end
+
+  defp get_page_size(page_size_string) when is_nil(page_size_string), do: @users_page_size
+
+  defp get_page_size(page_size_string) do
+    case Integer.parse(page_size_string) do
+      {page_size, _} -> page_size
+      :error -> @users_page_size
+    end
+  end
 end
diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs
index a3042fa05..42e0daf8e 100644
--- a/test/web/admin_api/admin_api_controller_test.exs
+++ b/test/web/admin_api/admin_api_controller_test.exs
@@ -342,7 +342,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
         |> get("/api/pleroma/admin/users?page=1")
 
       assert json_response(conn, 200) == %{
-               "count" => 1,
+               "count" => 2,
                "page_size" => 50,
                "users" => [
                  %{
@@ -369,7 +369,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
         |> get("/api/pleroma/admin/users?page=2")
 
       assert json_response(conn, 200) == %{
-               "count" => 1,
+               "count" => 2,
                "page_size" => 50,
                "users" => []
              }
@@ -416,9 +416,49 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
              }
     end
 
-    test "only local users" do
+    test "regular search with page size" do
       admin = insert(:user, info: %{is_admin: true})
       user = insert(:user, nickname: "bob")
+      user2 = insert(:user, nickname: "bo")
+
+      conn =
+        build_conn()
+        |> assign(:user, admin)
+        |> get("/api/pleroma/admin/users/search?query=bo&page_size=1&page=1")
+
+      assert json_response(conn, 200) == %{
+               "count" => 2,
+               "page_size" => 1,
+               "users" => [
+                 %{
+                   "deactivated" => user.info.deactivated,
+                   "id" => user.id,
+                   "nickname" => user.nickname
+                 }
+               ]
+             }
+
+      conn =
+        build_conn()
+        |> assign(:user, admin)
+        |> get("/api/pleroma/admin/users/search?query=bo&page_size=1&page=2")
+
+      assert json_response(conn, 200) == %{
+               "count" => 2,
+               "page_size" => 1,
+               "users" => [
+                 %{
+                   "deactivated" => user2.info.deactivated,
+                   "id" => user2.id,
+                   "nickname" => user2.nickname
+                 }
+               ]
+             }
+    end
+
+    test "only local users" do
+      admin = insert(:user, info: %{is_admin: true}, nickname: "john")
+      user = insert(:user, nickname: "bob")
 
       insert(:user, nickname: "bobb", local: false)
 

From bf30df99cb9a08870f7c6a25e0f9f3d2e03d5de7 Mon Sep 17 00:00:00 2001
From: Maxim Filippov <colixer@gmail.com>
Date: Sat, 2 Mar 2019 17:32:40 +0300
Subject: [PATCH 62/68] We do not guarantee the order of elements when we
 search

---
 test/user_test.exs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/user_test.exs b/test/user_test.exs
index 4f9de4e31..188ab1a5c 100644
--- a/test/user_test.exs
+++ b/test/user_test.exs
@@ -901,8 +901,8 @@ defmodule Pleroma.UserTest do
       {:ok, follower} = User.follow(follower, u1)
       {:ok, u1} = User.follow(u1, friend)
 
-      assert [friend.id, follower.id, u2.id] ==
-               Enum.map(User.search("doe", resolve: false, for_user: u1), & &1.id)
+      assert [friend.id, follower.id, u2.id] --
+               Enum.map(User.search("doe", resolve: false, for_user: u1), & &1.id) == []
     end
 
     test "finds a user whose name is nil" do

From 08c6aeeed7ff5db242050d06e707f99b6d75684d Mon Sep 17 00:00:00 2001
From: Maxim Filippov <colixer@gmail.com>
Date: Sat, 2 Mar 2019 17:32:46 +0300
Subject: [PATCH 63/68] Add docs

---
 docs/Admin-API.md | 49 ++++++++++++++++++++++++++++++-----------------
 1 file changed, 31 insertions(+), 18 deletions(-)

diff --git a/docs/Admin-API.md b/docs/Admin-API.md
index 4403620bf..407647645 100644
--- a/docs/Admin-API.md
+++ b/docs/Admin-API.md
@@ -7,20 +7,27 @@ Authentication is required and the user must be an admin.
 ### List users
 
 - Method `GET`
+- Params:
+  - `page`: **integer** page number
+  - `page_size`: **integer** number of users per page (default is `50`)
 - Response:
 
 ```JSON
-[
+{
+  "page_size": integer,
+  "count": integer,
+  "users": [
     {
-        "deactivated": bool,
-        "id": integer,
-        "nickname": string
+      "deactivated": bool,
+      "id": integer,
+      "nickname": string
     },
     ...
-]
+  ]
+}
 ```
 
-## `/api/pleroma/admin/users/search?query={query}&local={local}`
+## `/api/pleroma/admin/users/search?query={query}&local={local}&page={page}&page_size={page_size}`
 
 ### Search users by name or nickname
 
@@ -28,17 +35,23 @@ Authentication is required and the user must be an admin.
 - Params:
   - `query`: **string** search term
   - `local`: **bool** whether to return only local users
+  - `page`: **integer** page number
+  - `page_size`: **integer** number of users per page (default is `50`)
 - Response:
 
 ```JSON
-[
+{
+  "page_size": integer,
+  "count": integer,
+  "users": [
     {
-        "deactivated": bool,
-        "id": integer,
-        "nickname": string
+      "deactivated": bool,
+      "id": integer,
+      "nickname": string
     },
     ...
-]
+  ]
+}
 ```
 
 ## `/api/pleroma/admin/user`
@@ -70,9 +83,9 @@ Authentication is required and the user must be an admin.
 
 ```JSON
 {
-    "deactivated": bool,
-    "id": integer,
-    "nickname": string
+  "deactivated": bool,
+  "id": integer,
+  "nickname": string
 }
 ```
 
@@ -102,8 +115,8 @@ Authentication is required and the user must be an admin.
 
 ```JSON
 {
-    "is_moderator": bool,
-    "is_admin": bool
+  "is_moderator": bool,
+  "is_admin": bool
 }
 ```
 
@@ -119,8 +132,8 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
 
 ```JSON
 {
-    "is_moderator": bool,
-    "is_admin": bool
+  "is_moderator": bool,
+  "is_admin": bool
 }
 ```
 

From 1a1f4520cd711d46a53ffa0ec657f8a7e46896e7 Mon Sep 17 00:00:00 2001
From: eugenijm <eugenijm@protonmail.com>
Date: Sat, 2 Mar 2019 22:18:51 +0300
Subject: [PATCH 64/68] Use sql query in User.get_follow_requests/1 for
 filtering logic

---
 lib/pleroma/user.ex | 15 +++++++--------
 test/user_test.exs  | 14 ++++++++++++++
 2 files changed, 21 insertions(+), 8 deletions(-)

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index d58274508..06b430ccd 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -648,15 +648,14 @@ defmodule Pleroma.User do
   end
 
   def get_follow_requests(%User{} = user) do
-    q = get_follow_requests_query(user)
-    reqs = Repo.all(q)
-
     users =
-      Enum.map(reqs, fn req -> req.actor end)
-      |> Enum.uniq()
-      |> Enum.map(fn ap_id -> get_by_ap_id(ap_id) end)
-      |> Enum.filter(fn u -> !is_nil(u) end)
-      |> Enum.filter(fn u -> !following?(u, user) end)
+      user
+      |> User.get_follow_requests_query()
+      |> join(:inner, [a], u in User, a.actor == u.ap_id)
+      |> where([a, u], not fragment("? @> ?", u.following, ^[user.follower_address]))
+      |> group_by([a, u], u.id)
+      |> select([a, u], u)
+      |> Repo.all()
 
     {:ok, users}
   end
diff --git a/test/user_test.exs b/test/user_test.exs
index cbe4693fc..b8d41ecfd 100644
--- a/test/user_test.exs
+++ b/test/user_test.exs
@@ -64,6 +64,20 @@ defmodule Pleroma.UserTest do
     assert activity
   end
 
+  test "doesn't return already accepted or duplicate follow requests" do
+    locked = insert(:user, %{info: %{locked: true}})
+    pending_follower = insert(:user)
+    accepted_follower = insert(:user)
+
+    Pleroma.Web.TwitterAPI.TwitterAPI.follow(pending_follower, %{"user_id" => locked.id})
+    Pleroma.Web.TwitterAPI.TwitterAPI.follow(pending_follower, %{"user_id" => locked.id})
+    Pleroma.Web.TwitterAPI.TwitterAPI.follow(accepted_follower, %{"user_id" => locked.id})
+    User.maybe_follow(accepted_follower, locked)
+
+    assert {:ok, [activity]} = User.get_follow_requests(locked)
+    assert activity
+  end
+
   test "follow_all follows mutliple users" do
     user = insert(:user)
     followed_zero = insert(:user)

From c46950d3b16e6fe1ebb86a202ca47a810bfb76dc Mon Sep 17 00:00:00 2001
From: eugenijm <eugenijm@protonmail.com>
Date: Sun, 3 Mar 2019 13:21:03 +0300
Subject: [PATCH 65/68] Increment user note count only on public activities

---
 lib/pleroma/web/activity_pub/activity_pub.ex | 12 +++++-
 test/web/activity_pub/activity_pub_test.exs  | 43 ++++++++++++++++++++
 2 files changed, 53 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 16ae65867..4e2056e20 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -81,6 +81,14 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp check_remote_limit(_), do: true
 
+  def increase_note_count_if_public(actor, object) do
+    if is_public?(object), do: User.increase_note_count(actor), else: {:ok, actor}
+  end
+
+  def decrease_note_count_if_public(actor, object) do
+    if is_public?(object), do: User.decrease_note_count(actor), else: {:ok, actor}
+  end
+
   def insert(map, local \\ true) when is_map(map) do
     with nil <- Activity.normalize(map),
          map <- lazy_put_activity_defaults(map),
@@ -163,7 +171,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
            ),
          {:ok, activity} <- insert(create_data, local),
          # Changing note count prior to enqueuing federation task in order to avoid race conditions on updating user.info
-         {:ok, _actor} <- User.increase_note_count(actor),
+         {:ok, _actor} <- increase_note_count_if_public(actor, activity),
          :ok <- maybe_federate(activity) do
       {:ok, activity}
     end
@@ -316,7 +324,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     with {:ok, _} <- Object.delete(object),
          {:ok, activity} <- insert(data, local),
          # Changing note count prior to enqueuing federation task in order to avoid race conditions on updating user.info
-         {:ok, _actor} <- User.decrease_note_count(user),
+         {:ok, _actor} <- decrease_note_count_if_public(user, object),
          :ok <- maybe_federate(activity) do
       {:ok, activity}
     end
diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs
index ac3a565de..70a98824d 100644
--- a/test/web/activity_pub/activity_pub_test.exs
+++ b/test/web/activity_pub/activity_pub_test.exs
@@ -205,6 +205,25 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
       assert activity.actor == user.ap_id
       assert activity.recipients == ["user1", "user2", user.ap_id]
     end
+
+    test "increases user note count only for public activities" do
+      user = insert(:user)
+
+      {:ok, _} =
+        CommonAPI.post(Repo.get(User, user.id), %{"status" => "1", "visibility" => "public"})
+
+      {:ok, _} =
+        CommonAPI.post(Repo.get(User, user.id), %{"status" => "2", "visibility" => "unlisted"})
+
+      {:ok, _} =
+        CommonAPI.post(Repo.get(User, user.id), %{"status" => "2", "visibility" => "private"})
+
+      {:ok, _} =
+        CommonAPI.post(Repo.get(User, user.id), %{"status" => "3", "visibility" => "direct"})
+
+      user = Repo.get(User, user.id)
+      assert user.info.note_count == 2
+    end
   end
 
   describe "fetch activities for recipients" do
@@ -640,6 +659,30 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
 
       assert Repo.get(Object, object.id).data["type"] == "Tombstone"
     end
+
+    test "decrements user note count only for public activities" do
+      user = insert(:user, info: %{note_count: 10})
+
+      {:ok, a1} =
+        CommonAPI.post(Repo.get(User, user.id), %{"status" => "yeah", "visibility" => "public"})
+
+      {:ok, a2} =
+        CommonAPI.post(Repo.get(User, user.id), %{"status" => "yeah", "visibility" => "unlisted"})
+
+      {:ok, a3} =
+        CommonAPI.post(Repo.get(User, user.id), %{"status" => "yeah", "visibility" => "private"})
+
+      {:ok, a4} =
+        CommonAPI.post(Repo.get(User, user.id), %{"status" => "yeah", "visibility" => "direct"})
+
+      {:ok, _} = a1.data["object"]["id"] |> Object.get_by_ap_id() |> ActivityPub.delete()
+      {:ok, _} = a2.data["object"]["id"] |> Object.get_by_ap_id() |> ActivityPub.delete()
+      {:ok, _} = a3.data["object"]["id"] |> Object.get_by_ap_id() |> ActivityPub.delete()
+      {:ok, _} = a4.data["object"]["id"] |> Object.get_by_ap_id() |> ActivityPub.delete()
+
+      user = Repo.get(User, user.id)
+      assert user.info.note_count == 10
+    end
   end
 
   describe "timeline post-processing" do

From 8a1e0c9bee4173a7cd2c6b6174293097d78bea19 Mon Sep 17 00:00:00 2001
From: eugenijm <eugenijm@protonmail.com>
Date: Sun, 3 Mar 2019 15:35:32 +0300
Subject: [PATCH 66/68] Added migration to update existing user note counters

---
 ...190303120636_update_user_note_counters.exs | 41 +++++++++++++++++++
 1 file changed, 41 insertions(+)
 create mode 100644 priv/repo/migrations/20190303120636_update_user_note_counters.exs

diff --git a/priv/repo/migrations/20190303120636_update_user_note_counters.exs b/priv/repo/migrations/20190303120636_update_user_note_counters.exs
new file mode 100644
index 000000000..54e68f7c9
--- /dev/null
+++ b/priv/repo/migrations/20190303120636_update_user_note_counters.exs
@@ -0,0 +1,41 @@
+defmodule Pleroma.Repo.Migrations.UpdateUserNoteCounters do
+  use Ecto.Migration
+
+  @public "https://www.w3.org/ns/activitystreams#Public"
+
+  def up do
+    execute """
+      WITH public_note_count AS (
+        SELECT
+          data->>'actor' AS actor,
+          count(id) AS count
+        FROM objects
+        WHERE data->>'type' = 'Note' AND (
+          data->'cc' ? '#{@public}' OR data->'to' ? '#{@public}'
+        )
+        GROUP BY data->>'actor'
+      )
+      UPDATE users AS u
+      SET "info" = jsonb_set(u.info, '{note_count}', o.count::varchar::jsonb, true)
+      FROM public_note_count AS o
+      WHERE u.ap_id = o.actor
+    """
+  end
+
+  def down do
+    execute """
+      WITH public_note_count AS (
+        SELECT
+          data->>'actor' AS actor,
+          count(id) AS count
+        FROM objects
+        WHERE data->>'type' = 'Note'
+        GROUP BY data->>'actor'
+      )
+      UPDATE users AS u
+      SET "info" = jsonb_set(u.info, '{note_count}', o.count::varchar::jsonb, true)
+      FROM public_note_count AS o
+      WHERE u.ap_id = o.actor
+    """
+  end
+end

From af0039a3a0b7f466120211f2c81361311ecbcf02 Mon Sep 17 00:00:00 2001
From: eugenijm <eugenijm@protonmail.com>
Date: Sun, 3 Mar 2019 17:27:09 +0300
Subject: [PATCH 67/68] Use atomic update for note count and follower count

---
 lib/pleroma/user.ex | 87 +++++++++++++++++++++++++++++----------------
 1 file changed, 56 insertions(+), 31 deletions(-)

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index d58274508..115c03176 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -662,23 +662,43 @@ defmodule Pleroma.User do
   end
 
   def increase_note_count(%User{} = user) do
-    info_cng = User.Info.add_to_note_count(user.info, 1)
-
-    cng =
-      change(user)
-      |> put_embed(:info, info_cng)
-
-    update_and_set_cache(cng)
+    User
+    |> where(id: ^user.id)
+    |> update([u],
+      set: [
+        info:
+          fragment(
+            "jsonb_set(?, '{note_count}', ((?->>'note_count')::int + 1)::varchar::jsonb, true)",
+            u.info,
+            u.info
+          )
+      ]
+    )
+    |> Repo.update_all([], returning: true)
+    |> case do
+      {1, [user]} -> set_cache(user)
+      _ -> {:error, user}
+    end
   end
 
   def decrease_note_count(%User{} = user) do
-    info_cng = User.Info.add_to_note_count(user.info, -1)
-
-    cng =
-      change(user)
-      |> put_embed(:info, info_cng)
-
-    update_and_set_cache(cng)
+    User
+    |> where(id: ^user.id)
+    |> update([u],
+      set: [
+        info:
+          fragment(
+            "jsonb_set(?, '{note_count}', (greatest(0, (?->>'note_count')::int - 1))::varchar::jsonb, true)",
+            u.info,
+            u.info
+          )
+      ]
+    )
+    |> Repo.update_all([], returning: true)
+    |> case do
+      {1, [user]} -> set_cache(user)
+      _ -> {:error, user}
+    end
   end
 
   def update_note_count(%User{} = user) do
@@ -702,24 +722,29 @@ defmodule Pleroma.User do
 
   def update_follower_count(%User{} = user) do
     follower_count_query =
-      from(
-        u in User,
-        where: ^user.follower_address in u.following,
-        where: u.id != ^user.id,
-        select: count(u.id)
-      )
+      User
+      |> where([u], ^user.follower_address in u.following)
+      |> where([u], u.id != ^user.id)
+      |> select([u], %{count: count(u.id)})
 
-    follower_count = Repo.one(follower_count_query)
-
-    info_cng =
-      user.info
-      |> User.Info.set_follower_count(follower_count)
-
-    cng =
-      change(user)
-      |> put_embed(:info, info_cng)
-
-    update_and_set_cache(cng)
+    User
+    |> where(id: ^user.id)
+    |> join(:inner, [u], s in subquery(follower_count_query))
+    |> update([u, s],
+      set: [
+        info:
+          fragment(
+            "jsonb_set(?, '{follower_count}', ?::varchar::jsonb, true)",
+            u.info,
+            s.count
+          )
+      ]
+    )
+    |> Repo.update_all([], returning: true)
+    |> case do
+      {1, [user]} -> set_cache(user)
+      _ -> {:error, user}
+    end
   end
 
   def get_users_from_set_query(ap_ids, false) do

From d5418e9ff78785c48bc94fbc8cb146ffe90c1fc5 Mon Sep 17 00:00:00 2001
From: eugenijm <eugenijm@protonmail.com>
Date: Sun, 3 Mar 2019 18:39:37 +0300
Subject: [PATCH 68/68] Remove follow_request_count as it's not needed for FE
 anymore.

MastoFE uses `GET /api/v1/follow_requests` and PleromaFE uses
`GET /api/pleroma/friend_requests` which they query on the initial page
load.
---
 lib/pleroma/user.ex                           | 26 -------------------
 lib/pleroma/user/info.ex                      |  1 -
 lib/pleroma/web/activity_pub/activity_pub.ex  | 12 +++------
 .../web/twitter_api/views/user_view.ex        |  9 -------
 .../mastodon_api_controller_test.exs          |  4 ---
 .../twitter_api_controller_test.exs           |  5 ----
 6 files changed, 4 insertions(+), 53 deletions(-)

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index d58274508..0d2b838db 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -621,32 +621,6 @@ defmodule Pleroma.User do
     )
   end
 
-  def update_follow_request_count(%User{} = user) do
-    subquery =
-      user
-      |> User.get_follow_requests_query()
-      |> select([a], %{count: count(a.id)})
-
-    User
-    |> where(id: ^user.id)
-    |> join(:inner, [u], s in subquery(subquery))
-    |> update([u, s],
-      set: [
-        info:
-          fragment(
-            "jsonb_set(?, '{follow_request_count}', ?::varchar::jsonb, true)",
-            u.info,
-            s.count
-          )
-      ]
-    )
-    |> Repo.update_all([], returning: true)
-    |> case do
-      {1, [user]} -> {:ok, user}
-      _ -> {:error, user}
-    end
-  end
-
   def get_follow_requests(%User{} = user) do
     q = get_follow_requests_query(user)
     reqs = Repo.all(q)
diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex
index 00a0f6df3..818b64645 100644
--- a/lib/pleroma/user/info.ex
+++ b/lib/pleroma/user/info.ex
@@ -12,7 +12,6 @@ defmodule Pleroma.User.Info do
     field(:source_data, :map, default: %{})
     field(:note_count, :integer, default: 0)
     field(:follower_count, :integer, default: 0)
-    field(:follow_request_count, :integer, default: 0)
     field(:locked, :boolean, default: false)
     field(:confirmation_pending, :boolean, default: false)
     field(:confirmation_token, :string, default: nil)
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index b81198629..7282d4239 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -175,8 +175,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
     with data <- %{"to" => to, "type" => "Accept", "actor" => actor.ap_id, "object" => object},
          {:ok, activity} <- insert(data, local),
-         :ok <- maybe_federate(activity),
-         _ <- User.update_follow_request_count(actor) do
+         :ok <- maybe_federate(activity) do
       {:ok, activity}
     end
   end
@@ -187,8 +186,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
     with data <- %{"to" => to, "type" => "Reject", "actor" => actor.ap_id, "object" => object},
          {:ok, activity} <- insert(data, local),
-         :ok <- maybe_federate(activity),
-         _ <- User.update_follow_request_count(actor) do
+         :ok <- maybe_federate(activity) do
       {:ok, activity}
     end
   end
@@ -286,8 +284,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   def follow(follower, followed, activity_id \\ nil, local \\ true) do
     with data <- make_follow_data(follower, followed, activity_id),
          {:ok, activity} <- insert(data, local),
-         :ok <- maybe_federate(activity),
-         _ <- User.update_follow_request_count(followed) do
+         :ok <- maybe_federate(activity) do
       {:ok, activity}
     end
   end
@@ -297,8 +294,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
          {:ok, follow_activity} <- update_follow_state(follow_activity, "cancelled"),
          unfollow_data <- make_unfollow_data(follower, followed, follow_activity, activity_id),
          {:ok, activity} <- insert(unfollow_data, local),
-         :ok <- maybe_federate(activity),
-         _ <- User.update_follow_request_count(followed) do
+         :ok <- maybe_federate(activity) do
       {:ok, activity}
     end
   end
diff --git a/lib/pleroma/web/twitter_api/views/user_view.ex b/lib/pleroma/web/twitter_api/views/user_view.ex
index 22f33e0b5..e72ce977c 100644
--- a/lib/pleroma/web/twitter_api/views/user_view.ex
+++ b/lib/pleroma/web/twitter_api/views/user_view.ex
@@ -133,7 +133,6 @@ defmodule Pleroma.Web.TwitterAPI.UserView do
           "tags" => user.tags
         }
         |> maybe_with_activation_status(user, for_user)
-        |> maybe_with_follow_request_count(user, for_user)
     }
 
     data =
@@ -155,14 +154,6 @@ defmodule Pleroma.Web.TwitterAPI.UserView do
 
   defp maybe_with_activation_status(data, _, _), do: data
 
-  defp maybe_with_follow_request_count(data, %User{id: id, info: %{locked: true}} = user, %User{
-         id: id
-       }) do
-    Map.put(data, "follow_request_count", user.info.follow_request_count)
-  end
-
-  defp maybe_with_follow_request_count(data, _, _), do: data
-
   defp maybe_with_role(data, %User{id: id} = user, %User{id: id}) do
     Map.merge(data, %{"role" => role(user), "show_role" => user.info.show_role})
   end
diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs
index b52c2b805..f7f10662a 100644
--- a/test/web/mastodon_api/mastodon_api_controller_test.exs
+++ b/test/web/mastodon_api/mastodon_api_controller_test.exs
@@ -946,7 +946,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do
       other_user = Repo.get(User, other_user.id)
 
       assert User.following?(other_user, user) == false
-      assert user.info.follow_request_count == 1
 
       conn =
         build_conn()
@@ -960,7 +959,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do
       other_user = Repo.get(User, other_user.id)
 
       assert User.following?(other_user, user) == true
-      assert user.info.follow_request_count == 0
     end
 
     test "verify_credentials", %{conn: conn} do
@@ -982,7 +980,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do
       {:ok, _activity} = ActivityPub.follow(other_user, user)
 
       user = Repo.get(User, user.id)
-      assert user.info.follow_request_count == 1
 
       conn =
         build_conn()
@@ -996,7 +993,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do
       other_user = Repo.get(User, other_user.id)
 
       assert User.following?(other_user, user) == false
-      assert user.info.follow_request_count == 0
     end
   end
 
diff --git a/test/web/twitter_api/twitter_api_controller_test.exs b/test/web/twitter_api/twitter_api_controller_test.exs
index 7125d85ab..d18b65876 100644
--- a/test/web/twitter_api/twitter_api_controller_test.exs
+++ b/test/web/twitter_api/twitter_api_controller_test.exs
@@ -690,7 +690,6 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do
       followed = Repo.get(User, followed.id)
 
       refute User.ap_followers(followed) in current_user.following
-      assert followed.info.follow_request_count == 1
 
       assert json_response(conn, 200) ==
                UserView.render("show.json", %{user: followed, for: current_user})
@@ -1757,7 +1756,6 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do
       other_user = Repo.get(User, other_user.id)
 
       assert User.following?(other_user, user) == false
-      assert user.info.follow_request_count == 1
 
       conn =
         build_conn()
@@ -1769,7 +1767,6 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do
       assert relationship = json_response(conn, 200)
       assert other_user.id == relationship["id"]
       assert relationship["follows_you"] == true
-      assert user.info.follow_request_count == 0
     end
   end
 
@@ -1784,7 +1781,6 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do
       other_user = Repo.get(User, other_user.id)
 
       assert User.following?(other_user, user) == false
-      assert user.info.follow_request_count == 1
 
       conn =
         build_conn()
@@ -1796,7 +1792,6 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do
       assert relationship = json_response(conn, 200)
       assert other_user.id == relationship["id"]
       assert relationship["follows_you"] == false
-      assert user.info.follow_request_count == 0
     end
   end